You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maka.apache.org by wy...@apache.org on 2026/09/16 01:18:23 UTC
(maka) 04/29: feat(agent): resume handoffs with frozen execution state
This is an automated email from the ASF dual-hosted git repository.
M4n5ter pushed a commit to branch feat/runtime-host-rust
in repository https://gitbox.apache.org/repos/asf/maka.git
commit 51c6372e44ca63264b9532d542ff8fe183bbe047
Author: Wang <m4...@gmail.com>
AuthorDate: Tue Sep 15 13:25:06 2026 +0800
feat(agent): resume handoffs with frozen execution state
Generated-by: Codex
---
crates/agent/src/continuation.rs | 41 +++-
crates/agent/src/handoff.rs | 82 ++++---
crates/agent/src/lib.rs | 37 ++-
crates/agent/src/model_attempt.rs | 5 +-
crates/agent/src/runner.rs | 22 +-
crates/agent/src/steps.rs | 41 +++-
crates/agent/tests/integration/handoff.rs | 65 ++++-
.../agent/tests/integration/handoff_compaction.rs | 144 +++++++++++
crates/agent/tests/integration/main.rs | 1 +
.../agent/tests/integration/support/agent_loop.rs | 10 +-
crates/event-log/src/context/latest_main.rs | 3 +-
crates/event-log/src/context/read.rs | 22 ++
crates/event-log/src/handoff.rs | 30 +--
crates/event-log/src/handoff/budget.rs | 94 +++++++
crates/event-log/tests/continuation/handoff.rs | 18 ++
.../event-log/tests/continuation/handoff_budget.rs | 271 +++++++++++++--------
.../event-log/tests/continuation/handoff_claim.rs | 27 ++
crates/presentation/tests/projection.rs | 13 +
crates/runtime/src/handoff.rs | 10 +
crates/runtime/src/handoff/execution.rs | 72 ++++++
crates/tools/src/availability.rs | 24 ++
crates/tools/src/catalog.rs | 23 ++
crates/tools/src/dispatch.rs | 11 +
crates/tools/tests/integration/catalog.rs | 37 ++-
24 files changed, 908 insertions(+), 195 deletions(-)
diff --git a/crates/agent/src/continuation.rs b/crates/agent/src/continuation.rs
index f49387f0c4..e67220a27f 100644
--- a/crates/agent/src/continuation.rs
+++ b/crates/agent/src/continuation.rs
@@ -41,13 +41,24 @@ pub(super) async fn prepare(
cancellation: &CancellationToken,
) -> Result<ContinuationClaim, RunError> {
let (base, replay) = inspect(inner, input, source, catalog, cancellation).await?;
+ let pause = match &input.work {
+ crate::RunWork::Handoff { pause, .. } => Some(pause),
+ _ => None,
+ };
let claim = ContinuationClaim {
- id: uuid::Uuid::new_v4().to_string(),
+ id: pause.map_or_else(
+ || uuid::Uuid::new_v4().to_string(),
+ |pause| pause.intent.claim_id.clone(),
+ ),
source: source.clone(),
base,
replay,
};
- claim.validate(&input.invocation).map_err(invalid)?;
+ match pause {
+ Some(pause) => pause.validate_claim(&claim, &input.invocation),
+ None => claim.validate(&input.invocation),
+ }
+ .map_err(invalid)?;
Ok(claim)
}
@@ -101,6 +112,17 @@ pub(super) async fn inspect(
{
return Err(invalid("continuation workspace identity changed"));
}
+ if let crate::RunWork::Handoff { pause, .. } = &input.work
+ && (configuration != &input.configuration
+ || !matches!(
+ &prefix.events.last().expect("checked nonempty source").event.fact,
+ Fact::InvocationEnded {
+ outcome: maka_runtime::event::InvocationOutcome::HandoffPaused { pause: sealed }
+ } if sealed == pause.as_ref()
+ ))
+ {
+ return Err(invalid("handoff does not match its sealed source"));
+ }
let base = match opening {
InvocationInput::Message { .. } => {
let base = inner
@@ -127,7 +149,10 @@ pub(super) async fn inspect(
&context,
ModelPurpose::Main,
cancellation,
- Some(base.high_water),
+ match &input.work {
+ crate::RunWork::Handoff { pause, .. } => pause.execution.replay_base,
+ _ => Some(base.high_water),
+ },
)
.await?;
if !matches!(
@@ -149,10 +174,18 @@ pub(super) async fn inspect(
input.configuration.tool_mode,
inner.cells.clone(),
);
+ if let crate::RunWork::Handoff { pause, .. } = &input.work {
+ tools.restore(&pause.execution.tools)?;
+ }
let definitions = tools.capture().definitions();
let mut available: std::collections::HashSet<_> = catalog.names().into_iter().collect();
available.extend(definitions.iter().map(|definition| definition.name.clone()));
- for message in &prompt {
+ // A handoff replays settled facts, including rejected calls to unavailable
+ // tools. It never executes them again; the live catalog is checked above.
+ for message in prompt
+ .iter()
+ .filter(|_| !matches!(input.work, crate::RunWork::Handoff { .. }))
+ {
if let Message::Assistant { content, .. } = message {
for part in content {
if let AssistantPart::ToolCall {
diff --git a/crates/agent/src/handoff.rs b/crates/agent/src/handoff.rs
index c986e52c30..7c0f0ad891 100644
--- a/crates/agent/src/handoff.rs
+++ b/crates/agent/src/handoff.rs
@@ -18,7 +18,7 @@
*/
use maka_runtime::handoff::{HandoffIntent, HandoffPause};
-use std::{num::NonZeroU16, sync::Arc};
+use std::sync::Arc;
use tokio::sync::watch;
use tokio_util::sync::CancellationToken;
@@ -95,25 +95,23 @@ impl HandoffGate {
.ok_or(crate::RunError::Busy)
}
- pub(crate) async fn boundary<F: std::future::Future<Output = bool>>(
+ pub(crate) async fn boundary<F: std::future::Future<Output = Option<HandoffPause>>>(
&self,
- remaining_steps: NonZeroU16,
cancellation: &CancellationToken,
- preflight: impl FnOnce(HandoffPause) -> F,
+ preflight: impl FnOnce(HandoffIntent) -> F,
) -> Option<HandoffPause> {
let mut changes = self.state.subscribe();
let ticket = match &*changes.borrow_and_update() {
State::Requested(ticket) => ticket.clone(),
_ => return None,
};
- let pause = HandoffPause {
- intent: ticket.as_ref().clone(),
- remaining_steps,
- };
- if !preflight(pause.clone()).await || cancellation.is_cancelled() {
+ let pause = preflight(ticket.as_ref().clone()).await;
+ let Some(pause) =
+ pause.filter(|pause| pause.intent == *ticket && !cancellation.is_cancelled())
+ else {
self.cancel(&ticket);
return None;
- }
+ };
self.state.send_if_modified(|state| {
let State::Requested(intent) = state else {
return false;
@@ -252,6 +250,27 @@ impl PendingSeal {
mod tests {
use super::*;
+ fn prepared(intent: HandoffIntent) -> std::future::Ready<Option<HandoffPause>> {
+ use maka_runtime::handoff::{HandoffExecution, HandoffTools};
+ std::future::ready(Some(HandoffPause {
+ intent,
+ remaining_steps: std::num::NonZeroU16::new(2).unwrap(),
+ execution: Box::new(HandoffExecution {
+ route_identity: format!("sha256:{}", "b".repeat(64)),
+ context: None,
+ provider_options: serde_json::json!({}),
+ main_output_limit: None,
+ supports_vision: false,
+ tools: HandoffTools {
+ catalog_digest: format!("sha256:{}", "c".repeat(64)),
+ loaded: Default::default(),
+ },
+ compaction_attempted: false,
+ replay_base: None,
+ }),
+ }))
+ }
+
#[tokio::test]
async fn cancellation_prevents_commit_and_worker_exit_never_forges_a_seal() {
let source = maka_runtime::event::Invocation {
@@ -272,9 +291,7 @@ mod tests {
let cancellation = CancellationToken::new();
let reservation = gate.reserve(intent.clone()).unwrap();
let (result, held) = tokio::join!(
- gate.boundary(NonZeroU16::new(2).unwrap(), &cancellation, |_| {
- std::future::ready(false)
- }),
+ gate.boundary(&cancellation, |_| { std::future::ready(None) }),
reservation.ready(),
);
assert!(
@@ -282,40 +299,27 @@ mod tests {
"failed preflight releases the same Run"
);
let reservation = gate.reserve(intent.clone()).unwrap();
- let (result, ()) = tokio::join!(
- gate.boundary(NonZeroU16::new(2).unwrap(), &cancellation, |_| {
- std::future::ready(true)
- }),
- async {
- let held = reservation.ready().await.unwrap();
- cancellation.cancel();
- assert!(held.commit().is_none());
- },
- );
+ let (result, ()) = tokio::join!(gate.boundary(&cancellation, prepared), async {
+ let held = reservation.ready().await.unwrap();
+ cancellation.cancel();
+ assert!(held.commit().is_none());
+ },);
assert!(result.is_none());
let cancellation = CancellationToken::new();
let reservation = gate.reserve(intent.clone()).unwrap();
- let (result, stale) = tokio::join!(
- gate.boundary(NonZeroU16::new(2).unwrap(), &cancellation, |_| {
- std::future::ready(true)
- }),
- async {
- let held = reservation.ready().await.unwrap();
- cancellation.cancel();
- held
- },
- );
+ let (result, stale) = tokio::join!(gate.boundary(&cancellation, prepared), async {
+ let held = reservation.ready().await.unwrap();
+ cancellation.cancel();
+ held
+ },);
assert!(result.is_none());
let cancellation = CancellationToken::new();
let reservation = gate.reserve(intent).unwrap();
drop(stale); // The same durable intent does not reuse a scheduling owner.
- let (result, pending) = tokio::join!(
- gate.boundary(NonZeroU16::new(2).unwrap(), &cancellation, |_| {
- std::future::ready(true)
- }),
- async { reservation.ready().await.unwrap().commit().unwrap() },
- );
+ let (result, pending) = tokio::join!(gate.boundary(&cancellation, prepared), async {
+ reservation.ready().await.unwrap().commit().unwrap()
+ },);
assert!(result.is_some());
gate.close();
assert!(
diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rs
index 3e567210ac..7003fc2dcf 100644
--- a/crates/agent/src/lib.rs
+++ b/crates/agent/src/lib.rs
@@ -89,6 +89,11 @@ pub struct RunInput {
}
pub enum RunWork {
+ Handoff {
+ source: maka_runtime::continuation::RunBoundary,
+ pause: Box<maka_runtime::handoff::HandoffPause>,
+ tools: ToolCatalog,
+ },
Continuation {
source: maka_runtime::continuation::RunBoundary,
workhub_resume: Option<maka_runtime::workhub::ResumeOrigin>,
@@ -183,6 +188,23 @@ impl Engine {
.map_err(|reason| RunError::InvalidInput(reason.into()))?;
}
match &input.work {
+ RunWork::Handoff { source, pause, .. } => {
+ pause
+ .validate(&source.invocation)
+ .map_err(|reason| RunError::InvalidInput(reason.into()))?;
+ if input.invocation != pause.intent.successor(&source.invocation)
+ || input.request_fingerprint.is_some()
+ || input.context != pause.execution.context
+ || input.provider_options != pause.execution.provider_options
+ || input.main_output_limit != pause.execution.main_output_limit
+ || input.supports_vision != pause.execution.supports_vision
+ || model_attempt::route_identity(&input)? != pause.execution.route_identity
+ {
+ return Err(RunError::InvalidInput(
+ "handoff execution settings changed".into(),
+ ));
+ }
+ }
RunWork::Message {
message, max_steps, ..
} if *max_steps == 0 || *max_steps > 256 || message.text_bytes() > 64 * 1024 => {
@@ -252,8 +274,13 @@ impl Engine {
{
return Err(RunError::Busy);
}
- let handoff = (!matches!(input.work, RunWork::ContextCompact))
- .then(|| HandoffGate::new(input.invocation.clone(), input.invocation.run_id.clone()));
+ let handoff = (!matches!(input.work, RunWork::ContextCompact)).then(|| {
+ let root = match &input.work {
+ RunWork::Handoff { pause, .. } => &pause.intent.root_run_id,
+ _ => &input.invocation.run_id,
+ };
+ HandoffGate::new(input.invocation.clone(), root.clone())
+ });
let admission = Admission {
inner: self.0.clone(),
session_id,
@@ -265,9 +292,9 @@ impl Engine {
let inner = self.0.clone();
let invocation = input.invocation.clone();
let tool_names = Arc::new(match &input.work {
- RunWork::Message { tools, .. } | RunWork::Continuation { tools, .. } => {
- tools.names().into_iter().collect()
- }
+ RunWork::Message { tools, .. }
+ | RunWork::Continuation { tools, .. }
+ | RunWork::Handoff { tools, .. } => tools.names().into_iter().collect(),
RunWork::ContextCompact => Default::default(),
});
let worker_cancellation = cancellation.clone();
diff --git a/crates/agent/src/model_attempt.rs b/crates/agent/src/model_attempt.rs
index 5a3896e981..c79a585e95 100644
--- a/crates/agent/src/model_attempt.rs
+++ b/crates/agent/src/model_attempt.rs
@@ -156,7 +156,10 @@ pub(super) async fn execute(
_ = cancellation.cancelled() => return Err(RunError::Cancelled),
_ = tokio::time::sleep(delay) => {}
}
- if continuation_base.is_some() {
+ if matches!(
+ input.work,
+ crate::RunWork::Continuation { .. } | crate::RunWork::Handoff { .. }
+ ) {
let next = inner
.log
.read_model_context(
diff --git a/crates/agent/src/runner.rs b/crates/agent/src/runner.rs
index 56b21ef975..b6e0ac76c6 100644
--- a/crates/agent/src/runner.rs
+++ b/crates/agent/src/runner.rs
@@ -49,13 +49,20 @@ pub async fn run(
Err(error) => return Err(error.into()),
}
let claim = match &input.work {
- RunWork::Continuation { source, tools, .. } => {
+ RunWork::Continuation { source, tools, .. } | RunWork::Handoff { source, tools, .. } => {
Some(crate::continuation::prepare(&inner, &input, source, tools, &cancellation).await?)
}
_ => None,
};
- let continuation_base = claim.as_ref().map(|claim| claim.base.high_water);
+ let continuation_base = match &input.work {
+ RunWork::Handoff { pause, .. } => pause.execution.replay_base,
+ _ => claim.as_ref().map(|claim| claim.base.high_water),
+ };
let opening = match &input.work {
+ RunWork::Handoff { pause, .. } => InvocationInput::Handoff {
+ claim: Box::new(claim.expect("prepared handoff")),
+ pause: pause.clone(),
+ },
RunWork::Continuation { workhub_resume, .. } => InvocationInput::Continuation {
claim: Box::new(claim.expect("prepared continuation")),
workhub_resume: workhub_resume.clone(),
@@ -111,6 +118,17 @@ pub async fn run(
)
.await
.map(|outcome| (outcome, None)),
+ RunWork::Handoff { tools, pause, .. } => steps::run(
+ &inner,
+ &input,
+ tools,
+ usize::from(pause.remaining_steps.get()),
+ &cancellation,
+ continuation_base,
+ handoff.as_ref().expect("model Runs own a handoff gate"),
+ )
+ .await
+ .map(|outcome| (outcome, None)),
RunWork::ContextCompact => compact::run(
&inner,
&input,
diff --git a/crates/agent/src/steps.rs b/crates/agent/src/steps.rs
index 8afce8ae82..5ee315a11e 100644
--- a/crates/agent/src/steps.rs
+++ b/crates/agent/src/steps.rs
@@ -42,25 +42,42 @@ pub(super) async fn run(
inner.cells.clone(),
);
let mut attempted = false;
+ // This tracks work after this physical opening, not after the logical root.
+ // A successor may compact the sealed prefix with a PreTurn boundary.
let mut completed_step = false;
+ if let crate::RunWork::Handoff { pause, .. } = &input.work {
+ tools.restore(&pause.execution.tools)?;
+ attempted = pause.execution.compaction_attempted;
+ }
for step in 0..max_steps {
if cancellation.is_cancelled() {
return Err(RunError::Cancelled);
}
inner.log.commit_pending_steering(&input.invocation).await?;
if let Some(pause) = handoff
- .boundary(
- std::num::NonZeroU16::new((max_steps - step) as u16)
- .expect("validated step budget"),
- cancellation,
- |pause| async move {
- inner
- .log
- .check_handoff(&input.invocation, &pause)
- .await
- .is_ok()
- },
- )
+ .boundary(cancellation, |intent| async {
+ let pause = maka_runtime::handoff::HandoffPause {
+ intent,
+ remaining_steps: std::num::NonZeroU16::new((max_steps - step) as u16)
+ .expect("validated step budget"),
+ execution: Box::new(maka_runtime::handoff::HandoffExecution {
+ route_identity: model_attempt::route_identity(input).ok()?,
+ context: input.context.clone(),
+ provider_options: input.provider_options.clone(),
+ main_output_limit: input.main_output_limit,
+ supports_vision: input.supports_vision,
+ tools: tools.checkpoint(),
+ compaction_attempted: attempted,
+ replay_base: continuation_base,
+ }),
+ };
+ inner
+ .log
+ .check_handoff(&input.invocation, &pause)
+ .await
+ .ok()?;
+ Some(pause)
+ })
.await
{
return Ok(maka_runtime::event::InvocationOutcome::HandoffPaused { pause });
diff --git a/crates/agent/tests/integration/handoff.rs b/crates/agent/tests/integration/handoff.rs
index 97b0c5ef95..cfb824ffd3 100644
--- a/crates/agent/tests/integration/handoff.rs
+++ b/crates/agent/tests/integration/handoff.rs
@@ -62,6 +62,7 @@ async fn handoff_waits_for_settlement_rollback_keeps_run_and_seal_survives_reope
let (release_first, released_first) = oneshot::channel();
let (second_seen, second) = oneshot::channel();
let (release_second, released_second) = oneshot::channel();
+ let (third_seen, mut third) = oneshot::channel();
let server = tokio::spawn(async move {
let mut requests = Vec::new();
for (seen, release) in [(first_seen, released_first), (second_seen, released_second)] {
@@ -69,8 +70,13 @@ async fn handoff_waits_for_settlement_rollback_keeps_run_and_seal_survives_reope
requests.push(read_request(&mut socket).await);
seen.send(()).unwrap();
release.await.unwrap();
- respond(&mut socket, true).await;
+ crate::support::agent_loop::respond_tool(&mut socket,
+ Some(if requests.len() == 1 { "echo" } else { "unavailable" })).await;
}
+ let (mut socket, _) = listener.accept().await.unwrap();
+ requests.push(read_request(&mut socket).await);
+ third_seen.send(()).unwrap();
+ respond(&mut socket, false).await;
requests
});
let log = Arc::new(EventLog::open(&path).await.unwrap());
@@ -81,6 +87,8 @@ async fn handoff_waits_for_settlement_rollback_keeps_run_and_seal_survives_reope
let mut run = input(&base, "first", effect);
run.configuration.workspace_identity = Some(maka_runtime::execution::WorkspaceIdentity::from_marker_id(
"ef751105-55b5-4d65-a364-646281586a17").unwrap());
+ run.main_output_limit = Some(120);
+ let configuration = run.configuration.clone();
let running = engine.start(run, CancellationToken::new()).await.unwrap();
first.await.unwrap();
let gate = running.handoff().unwrap().clone();
@@ -107,10 +115,9 @@ async fn handoff_waits_for_settlement_rollback_keeps_run_and_seal_survives_reope
assert_eq!(receipt, expected);
running.wait().await.unwrap();
engine.drain().await;
- assert_eq!(count.load(Ordering::SeqCst), 2, "neither rollback nor seal repeats effects");
- let requests = server.await.unwrap();
- assert_eq!(requests.len(), 2, "no model request follows the committed hold");
- assert!(requests[1]["messages"].as_array().unwrap().iter().any(|m| m["role"] == "tool"));
+ assert_eq!(count.load(Ordering::SeqCst), 1, "rejected calls, rollback and seal never repeat effects");
+ assert!(tokio::time::timeout(Duration::from_millis(30), &mut third).await.is_err(),
+ "no model request follows the committed hold");
let before = log.prefix(100, 128 * 1024).await.unwrap();
assert_eq!(before.events.iter().filter(|e| matches!(e.event.fact, Fact::InvocationOpened { .. })).count(), 1);
assert!(matches!(&before.events.last().unwrap().event.fact,
@@ -122,11 +129,57 @@ async fn handoff_waits_for_settlement_rollback_keeps_run_and_seal_survives_reope
log.shutdown().await.unwrap();
drop(engine);
drop(log);
- let reopened = EventLog::open(&path).await.unwrap();
+ let reopened = Arc::new(EventLog::open(&path).await.unwrap());
assert_eq!(maka_agent::recovery::recover(&reopened).await.unwrap(), 0);
let after = reopened.prefix(100, 128 * 1024).await.unwrap();
assert_eq!(before.digest, after.digest);
assert_eq!(after.project_invocation("invocation-first").terminal, Some(TerminalStatus::Paused));
+ let prefix = reopened.run_prefix("session", "run-first", None, 100, 128 * 1024).await.unwrap().unwrap();
+ let source = maka_runtime::continuation::RunBoundary {
+ invocation: prefix.invocation,
+ high_water: prefix.high_water,
+ digest: prefix.digest,
+ };
+ let resumed_engine = Engine::new(reopened.clone(), ModelExecutor::new(1, Duration::from_secs(5)).unwrap(),
+ CodeExecutor::new(1, CellLimits::default()).unwrap());
+ let successor = || {
+ let effect = Arc::new(Effect { log: reopened.clone(), count: count.clone() });
+ let mut run = input(&base, "successor", effect);
+ let maka_agent::RunWork::Message { tools, .. } = run.work else { unreachable!() };
+ run.invocation = expected.intent.successor(&source.invocation);
+ run.configuration = configuration.clone();
+ run.main_output_limit = expected.execution.main_output_limit;
+ run.work = maka_agent::RunWork::Handoff {
+ source: source.clone(), pause: Box::new(expected.clone()), tools,
+ };
+ run
+ };
+ for field in 0..5 {
+ let mut changed = successor();
+ match field {
+ 0 => changed.main_output_limit = None,
+ 1 => changed.provider_options = serde_json::json!({"changed": true}),
+ 2 => changed.provider.model = "changed".into(),
+ 3 => changed.configuration.cwd.push_str("/changed"),
+ _ => {
+ let maka_agent::RunWork::Handoff { tools, .. } = &mut changed.work else { unreachable!() };
+ *tools = Default::default();
+ },
+ }
+ assert!(resumed_engine.start(changed, CancellationToken::new()).await.is_err());
+ assert_eq!(reopened.prefix(100, 128 * 1024).await.unwrap().digest, after.digest);
+ }
+ resumed_engine.run(successor(), CancellationToken::new()).await.unwrap();
+ resumed_engine.drain().await;
+ let requests = server.await.unwrap();
+ assert_eq!(requests.len(), 3);
+ let messages = requests[2]["messages"].as_array().unwrap();
+ assert_eq!(messages.iter().filter(|m| m["role"] == "user").count(), 1);
+ assert_eq!(messages.iter().filter(|m| m["role"] == "tool").count(), 2);
+ assert_eq!(count.load(Ordering::SeqCst), 1, "settled effects are replayed as facts, never executed");
+ let finished = reopened.prefix(100, 128 * 1024).await.unwrap();
+ assert_eq!(finished.project_invocation("invocation-committed").terminal, Some(TerminalStatus::Completed));
+ assert!(!serde_json::to_string(&finished.events).unwrap().contains("do-not-persist-this-secret"));
reopened.shutdown().await.unwrap();
}).await.unwrap();
}
diff --git a/crates/agent/tests/integration/handoff_compaction.rs b/crates/agent/tests/integration/handoff_compaction.rs
new file mode 100644
index 0000000000..f69f626b55
--- /dev/null
+++ b/crates/agent/tests/integration/handoff_compaction.rs
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::support::{
+ agent_loop::{Effect, input, respond},
+ context as fixture,
+ http::read_request,
+};
+use maka_agent::RunWork;
+use maka_event_log::{EventLog, context::LatestMainContext};
+use maka_runtime::{
+ context::{CheckpointMode, ModelRequestContext},
+ continuation::RunBoundary,
+ event::Fact,
+ execution::{ModelBinding, WorkspaceIdentity},
+ handoff::HandoffIntent,
+};
+use std::{
+ sync::{
+ Arc,
+ atomic::{AtomicUsize, Ordering},
+ },
+ time::Duration,
+};
+use tokio::{io::AsyncWriteExt, net::TcpListener, sync::oneshot};
+use tokio_util::sync::CancellationToken;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn handoff_preserves_conversation_replay_and_compacts_before_its_physical_anchor() {
+ tokio::time::timeout(Duration::from_secs(20), async {
+ let directory = tempfile::tempdir().unwrap();
+ let log = Arc::new(EventLog::open(&directory.path().join("events.sqlite")).await.unwrap());
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let base = format!("http://{}/v1", listener.local_addr().unwrap());
+ let (seen, ready) = oneshot::channel();
+ let (release, released) = oneshot::channel();
+ let server = tokio::spawn(async move {
+ let (mut socket, _) = listener.accept().await.unwrap();
+ read_request(&mut socket).await;
+ let chunk = serde_json::json!({"id":"old","object":"chat.completion.chunk","created":1,"model":"old",
+ "choices":[{"index":0,"delta":{"reasoning_content":"old-route-thinking","content":"old-answer"},"finish_reason":"stop"}],
+ "usage":{"prompt_tokens":4,"completion_tokens":3,"total_tokens":7}});
+ let body = format!("data: {chunk}\n\ndata: [DONE]\n\n");
+ socket.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).as_bytes()).await.unwrap();
+ let (mut socket, _) = listener.accept().await.unwrap();
+ read_request(&mut socket).await;
+ seen.send(()).unwrap();
+ released.await.unwrap();
+ respond(&mut socket, true).await;
+ let (mut socket, _) = listener.accept().await.unwrap();
+ let summary = read_request(&mut socket).await;
+ fixture::respond(&mut socket, fixture::SUMMARY, "stop").await;
+ let (mut socket, _) = listener.accept().await.unwrap();
+ let main = read_request(&mut socket).await;
+ let failure = r#"{"error":{"message":"retry this request","type":"rate_limit_error"}}"#;
+ socket.write_all(format!("HTTP/1.1 429 Too Many Requests\r\nRetry-After: 0.01\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{failure}", failure.len()).as_bytes()).await.unwrap();
+ let (mut socket, _) = listener.accept().await.unwrap();
+ let retried = read_request(&mut socket).await;
+ assert_eq!(retried, main, "physical retries preserve the frozen request surface");
+ respond(&mut socket, false).await;
+ (summary, main)
+ });
+ let engine = fixture::engine(log.clone());
+ let mut prior = fixture::input(&base, "prior", false);
+ prior.provider.kind = maka_model::ProviderKind::OpenaiCompatible { name: "fixture".into() };
+ prior.provider.model = "old".into();
+ prior.configuration.model.as_mut().unwrap().model = "old".into();
+ engine.run(prior, CancellationToken::new()).await.unwrap();
+ let count = Arc::new(AtomicUsize::new(0));
+ let effect = Arc::new(Effect { log: log.clone(), count: count.clone() });
+ let mut first = input(&base, "first", effect.clone());
+ first.configuration.workspace_identity = Some(WorkspaceIdentity::from_marker_id(
+ "ef751105-55b5-4d65-a364-646281586a17").unwrap());
+ first.configuration.model = Some(ModelBinding {
+ connection_id: "connection".into(), connection_slug: "fixture".into(), model: "test".into(),
+ });
+ first.context = Some(ModelRequestContext {
+ provider_id: "openai".into(), context_window: Some(10), declared_window: Some(10),
+ });
+ let configuration = first.configuration.clone();
+ let context = first.context.clone();
+ let source_invocation = first.invocation.clone();
+ let running = engine.start(first, CancellationToken::new()).await.unwrap();
+ ready.await.unwrap();
+ let reservation = running.handoff().unwrap().reserve(HandoffIntent {
+ handoff_id: "compact".into(), host_epoch: "host".into(), root_run_id: "run-first".into(),
+ successor_run_id: "successor-run".into(), successor_invocation_id: "successor-invocation".into(),
+ claim_id: "claim".into(),
+ }).unwrap();
+ release.send(()).unwrap();
+ let pause = reservation.ready().await.unwrap().commit().unwrap().wait().await.unwrap();
+ assert!(!pause.execution.compaction_attempted);
+ assert_eq!(pause.execution.replay_base, None);
+ running.wait().await.unwrap();
+ engine.drain().await;
+ let prefix = log.run_prefix("session", "run-first", None, 100, 128 * 1024).await.unwrap().unwrap();
+ let mut successor = input(&base, "next", effect);
+ let RunWork::Message { tools, .. } = successor.work else { unreachable!() };
+ successor.invocation = pause.intent.successor(&source_invocation);
+ successor.configuration = configuration;
+ successor.context = context;
+ successor.work = RunWork::Handoff {
+ source: RunBoundary { invocation: prefix.invocation, high_water: prefix.high_water, digest: prefix.digest },
+ pause: Box::new(pause), tools,
+ };
+ let resumed = fixture::engine(log.clone());
+ resumed.run(successor, CancellationToken::new()).await.unwrap();
+ resumed.drain().await;
+ let (summary, main) = server.await.unwrap();
+ assert!(summary["messages"].to_string().contains("question first"));
+ assert!(main["messages"].to_string().contains("summary-marker"));
+ assert!(!main["messages"].to_string().contains("old-answer"));
+ assert_eq!(count.load(Ordering::SeqCst), 1);
+ let prefix = log.prefix(100, 256 * 1024).await.unwrap();
+ assert!(prefix.events.iter().any(|e| matches!(&e.event.fact,
+ Fact::ModelCompleted { output, .. } if output.parts.iter().any(|part| matches!(part,
+ maka_runtime::model::ModelPart::Text { text_kind: maka_runtime::model::TextKind::Thinking, .. })))));
+ assert!(prefix.events.iter().any(|e| matches!(&e.event.fact,
+ Fact::ContextCheckpointRecorded { checkpoint } if checkpoint.mode == CheckpointMode::PreTurn)));
+ let latest_sequence = prefix.events.iter().rev().find(|e|
+ e.event.invocation.invocation_id == "successor-invocation"
+ && matches!(e.event.fact, Fact::ModelCompleted { .. })).unwrap().sequence;
+ let source = log.read_model_context("session", None, 100, 256 * 1024).await.unwrap();
+ assert!(matches!(source.latest_main, LatestMainContext::Selected(latest)
+ if latest.sequence == latest_sequence));
+ log.shutdown().await.unwrap();
+ }).await.unwrap();
+}
diff --git a/crates/agent/tests/integration/main.rs b/crates/agent/tests/integration/main.rs
index c858dea6b3..05c30af1f3 100644
--- a/crates/agent/tests/integration/main.rs
+++ b/crates/agent/tests/integration/main.rs
@@ -28,6 +28,7 @@ mod context_stop;
mod dynamic_tools;
mod frozen_projection;
mod handoff;
+mod handoff_compaction;
mod image_projection;
mod lifetime;
mod model_interruption;
diff --git a/crates/agent/tests/integration/support/agent_loop.rs b/crates/agent/tests/integration/support/agent_loop.rs
index 6c92b75dc1..6ba15ef783 100644
--- a/crates/agent/tests/integration/support/agent_loop.rs
+++ b/crates/agent/tests/integration/support/agent_loop.rs
@@ -181,14 +181,18 @@ pub(crate) async fn enqueue(log: &EventLog, invocation: Invocation) {
}
pub(crate) async fn respond(socket: &mut tokio::net::TcpStream, tool: bool) {
+ respond_tool(socket, tool.then_some("echo")).await;
+}
+
+pub(crate) async fn respond_tool(socket: &mut tokio::net::TcpStream, tool: Option<&str>) {
use tokio::io::AsyncWriteExt;
- let delta = if tool {
- json!({"tool_calls":[{"index":0,"id":"call-1","type":"function","function":{"name":"echo","arguments":"{\"value\":42}"}}]})
+ let delta = if let Some(name) = tool {
+ json!({"tool_calls":[{"index":0,"id":"call-1","type":"function","function":{"name":name,"arguments":"{\"value\":42}"}}]})
} else {
json!({"content":"done"})
};
let first = json!({"id":"reply","object":"chat.completion.chunk","created":1,"model":"test","choices":[{"index":0,"delta":delta,"finish_reason":null}]});
- let last = json!({"id":"reply","object":"chat.completion.chunk","created":1,"model":"test","choices":[{"index":0,"delta":{},"finish_reason":if tool {"tool_calls"} else {"stop"}}],"usage":{"prompt_tokens":4,"completion_tokens":3,"total_tokens":7}});
+ let last = json!({"id":"reply","object":"chat.completion.chunk","created":1,"model":"test","choices":[{"index":0,"delta":{},"finish_reason":if tool.is_some() {"tool_calls"} else {"stop"}}],"usage":{"prompt_tokens":4,"completion_tokens":3,"total_tokens":7}});
let body = format!("data: {first}\n\ndata: {last}\n\ndata: [DONE]\n\n");
socket.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).as_bytes()).await.unwrap();
}
diff --git a/crates/event-log/src/context/latest_main.rs b/crates/event-log/src/context/latest_main.rs
index 33b3ebb936..16512550a8 100644
--- a/crates/event-log/src/context/latest_main.rs
+++ b/crates/event-log/src/context/latest_main.rs
@@ -129,7 +129,8 @@ pub(super) async fn read_selected(
let purpose: Option<&str> = row.try_get("purpose").ok()?;
let purpose = match (kind, purpose) {
("message" | "continuation", None | Some("main")) => ModelPurpose::Main,
- ("message" | "continuation" | "context_compact", Some("summary"))
+ ("handoff", Some("main")) => ModelPurpose::Main,
+ ("message" | "continuation" | "handoff" | "context_compact", Some("summary"))
| ("context_compact", None) => ModelPurpose::Summary,
_ => return None,
};
diff --git a/crates/event-log/src/context/read.rs b/crates/event-log/src/context/read.rs
index 3787001cfd..581237655c 100644
--- a/crates/event-log/src/context/read.rs
+++ b/crates/event-log/src/context/read.rs
@@ -104,6 +104,28 @@ impl EventLog {
}
}
+/// Use the same effective-tail/anchor accounting as the successor's first read.
+pub(crate) async fn check_handoff_capacity(
+ connection: &mut SqliteConnection,
+ opening: &maka_runtime::event::RuntimeEvent,
+ max_events: usize,
+ max_bytes: usize,
+) -> Result<(), StoreError> {
+ let selection = Selection::for_opening(connection, opening).await?;
+ let high = selection.high_water(connection, i64::MAX as u64).await?;
+ materialize_selected(
+ connection,
+ &selection,
+ high,
+ i64::MAX as u64,
+ max_events,
+ max_bytes,
+ super::LatestMainContext::TraceUnavailable,
+ )
+ .await?;
+ Ok(())
+}
+
pub(super) async fn materialize_selected(
connection: &mut SqliteConnection,
selection: &Selection,
diff --git a/crates/event-log/src/handoff.rs b/crates/event-log/src/handoff.rs
index f8ffb5fc7f..d2d8e17e8d 100644
--- a/crates/event-log/src/handoff.rs
+++ b/crates/event-log/src/handoff.rs
@@ -21,6 +21,8 @@ use crate::{EventLog, StoreError};
use maka_runtime::event::{Fact, InvocationInput, InvocationOutcome, RuntimeEvent};
use sqlx::{Connection, SqliteConnection};
+mod budget;
+
impl EventLog {
/// Eligibility only. The source still owns execution until its seal commits.
pub async fn check_handoff(
@@ -182,6 +184,14 @@ pub(crate) async fn validate(
));
}
};
+ let replay_base = match input {
+ InvocationInput::Continuation { claim, .. } => Some(claim.base.high_water),
+ InvocationInput::Handoff { pause, .. } => pause.execution.replay_base,
+ _ => None,
+ };
+ if pause.execution.replay_base != replay_base {
+ return Err(invalid("handoff changes its admitted history projection"));
+ }
if &pause.intent.root_run_id != root {
return Err(invalid(
"handoff must preserve its admitted logical model Run",
@@ -192,25 +202,7 @@ pub(crate) async fn validate(
crate::context::safety::settled_boundary(tx, &event.invocation.invocation_id, i64::MAX as u64)
.await?;
crate::interactions::lifecycle::require_closed(tx, &event.invocation).await?;
- // Use a stable upper bound in both preflight and append: time advancing
- // while held must not consume the headroom a second time.
- let mut envelope = event.clone();
- envelope.recorded_at = std::time::UNIX_EPOCH;
- let seal_bytes =
- serde_json::to_vec(&envelope)?.len() + (u64::MAX.ilog10() + u32::MAX.ilog10()) as usize;
- let source_bytes = maka_runtime::continuation::MAX_SOURCE_BYTES
- .checked_sub(seal_bytes)
- .ok_or(StoreError::PrefixTooLarge)?;
- crate::run_prefix::read(
- tx,
- &event.invocation.session_id,
- &event.invocation.run_id,
- &event.invocation.invocation_id,
- i64::MAX,
- maka_runtime::continuation::MAX_SOURCE_EVENTS - 1,
- source_bytes,
- )
- .await?;
+ budget::check(tx, event, &opening, pause).await?;
let occupied: bool = sqlx::query_scalar(
"SELECT EXISTS(SELECT 1 FROM runtime_events WHERE kind='invocation_opened'
AND (invocation_id=?1 OR json_extract(event_json,'$.invocation.run_id')=?2
diff --git a/crates/event-log/src/handoff/budget.rs b/crates/event-log/src/handoff/budget.rs
new file mode 100644
index 0000000000..73a92af124
--- /dev/null
+++ b/crates/event-log/src/handoff/budget.rs
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::StoreError;
+use maka_runtime::{
+ continuation::{
+ ContinuationClaim, MAX_SOURCE_BYTES, MAX_SOURCE_EVENTS, REPLAY_VERSION, ReplayEvidence,
+ RunBoundary, SessionBase,
+ },
+ event::{Fact, InvocationInput, RuntimeEvent},
+ handoff::HandoffPause,
+};
+use sqlx::SqliteConnection;
+
+pub(super) async fn check(
+ tx: &mut SqliteConnection,
+ seal: &RuntimeEvent,
+ opening: &RuntimeEvent,
+ pause: &HandoffPause,
+) -> Result<(), StoreError> {
+ let seal_bytes = envelope_bytes(seal)?;
+ crate::run_prefix::read(
+ tx,
+ &seal.invocation.session_id,
+ &seal.invocation.run_id,
+ &seal.invocation.invocation_id,
+ i64::MAX,
+ MAX_SOURCE_EVENTS - 1,
+ MAX_SOURCE_BYTES
+ .checked_sub(seal_bytes)
+ .ok_or(StoreError::PrefixTooLarge)?,
+ )
+ .await?;
+ let Fact::InvocationOpened { configuration, .. } = &opening.fact else {
+ return Err(super::invalid(
+ "handoff budget requires its canonical opening",
+ ));
+ };
+ // Sizing only: no claim is acquired and no guessed proof is persisted.
+ // Reserve the widest ordinals, fixed-width digests and both new envelopes.
+ let digest = format!("sha256:{}", "0".repeat(64));
+ let successor = RuntimeEvent::new(
+ pause.intent.successor(&seal.invocation),
+ Fact::InvocationOpened {
+ configuration: configuration.clone(),
+ input: InvocationInput::Handoff {
+ pause: Box::new(pause.clone()),
+ claim: Box::new(ContinuationClaim {
+ id: pause.intent.claim_id.clone(),
+ source: RunBoundary {
+ invocation: seal.invocation.clone(),
+ high_water: u64::MAX,
+ digest: digest.clone(),
+ },
+ base: SessionBase {
+ high_water: u64::MAX,
+ digest: digest.clone(),
+ },
+ replay: ReplayEvidence {
+ version: REPLAY_VERSION,
+ digest,
+ route_identity: pause.execution.route_identity.clone(),
+ },
+ }),
+ },
+ },
+ );
+ let bytes = MAX_SOURCE_BYTES
+ .checked_sub(seal_bytes + envelope_bytes(&successor)?)
+ .ok_or(StoreError::PrefixTooLarge)?;
+ crate::context::read::check_handoff_capacity(tx, opening, MAX_SOURCE_EVENTS - 2, bytes).await
+}
+
+fn envelope_bytes(event: &RuntimeEvent) -> Result<usize, StoreError> {
+ let mut envelope = event.clone();
+ envelope.recorded_at = std::time::UNIX_EPOCH;
+ Ok(serde_json::to_vec(&envelope)?.len() + (u64::MAX.ilog10() + u32::MAX.ilog10()) as usize)
+}
diff --git a/crates/event-log/tests/continuation/handoff.rs b/crates/event-log/tests/continuation/handoff.rs
index 06dbd3f76c..c8ebd3efef 100644
--- a/crates/event-log/tests/continuation/handoff.rs
+++ b/crates/event-log/tests/continuation/handoff.rs
@@ -26,6 +26,23 @@ use maka_runtime::{
};
use std::num::NonZeroU16;
+pub(super) fn execution() -> Box<maka_runtime::handoff::HandoffExecution> {
+ use maka_runtime::handoff::{HandoffExecution, HandoffTools};
+ Box::new(HandoffExecution {
+ route_identity: digest('b'),
+ context: None,
+ provider_options: serde_json::json!({}),
+ main_output_limit: None,
+ supports_vision: false,
+ tools: HandoffTools {
+ catalog_digest: digest('c'),
+ loaded: Default::default(),
+ },
+ compaction_attempted: false,
+ replay_base: None,
+ })
+}
+
#[tokio::test]
async fn handoff_rejects_unresolved_provider_effects_but_accepts_effect_free_summary_failure() {
for provider_effect in [false, true] {
@@ -85,6 +102,7 @@ async fn handoff_rejects_unresolved_provider_effects_but_accepts_effect_free_sum
claim_id: "claim".into(),
},
remaining_steps: NonZeroU16::new(2).unwrap(),
+ execution: execution(),
},
},
},
diff --git a/crates/event-log/tests/continuation/handoff_budget.rs b/crates/event-log/tests/continuation/handoff_budget.rs
index 6b05595248..40f7a813d9 100644
--- a/crates/event-log/tests/continuation/handoff_budget.rs
+++ b/crates/event-log/tests/continuation/handoff_budget.rs
@@ -29,113 +29,180 @@ use std::{
};
#[tokio::test]
-async fn near_capacity_handoff_keeps_its_seal_budget_when_timestamp_width_changes() {
- let directory = tempfile::tempdir().unwrap();
- let log = EventLog::open(&directory.path().join("capacity.sqlite"))
- .await
- .unwrap();
- let source = opening("root", None);
- append(&log, &source).await;
- let context = log
- .read_model_context("session", Some("root"), 100, 65536)
- .await
- .unwrap();
- append(
- &log,
- &RuntimeEvent::new(
+async fn handoff_reserves_both_envelopes_before_sealing_a_near_capacity_context() {
+ for reserve_successor in [false, true] {
+ let directory = tempfile::tempdir().unwrap();
+ let log = EventLog::open(&directory.path().join("capacity.sqlite"))
+ .await
+ .unwrap();
+ let source = opening("root", None);
+ append(&log, &source).await;
+ let context = log
+ .read_model_context("session", Some("root"), 100, 65536)
+ .await
+ .unwrap();
+ append(
+ &log,
+ &RuntimeEvent::new(
+ source.invocation.clone(),
+ Fact::ModelRequested {
+ step_id: "step".into(),
+ model_id: "test".into(),
+ purpose: Some(maka_runtime::context::ModelPurpose::Main),
+ context: None,
+ source_scope: context.source_evidence.scope,
+ source_high_water: context.source_evidence.high_water,
+ source_digest: context.source_evidence.digest,
+ effective_source_digest: Some(context.effective_source_digest),
+ input_digest: digest('a'),
+ route_identity: digest('b'),
+ checkpoint_event_id: None,
+ },
+ ),
+ )
+ .await;
+ let prefix = log
+ .run_prefix("session", "root", None, 100, MAX_SOURCE_BYTES)
+ .await
+ .unwrap()
+ .unwrap();
+ let existing_bytes: usize = prefix
+ .events
+ .iter()
+ .map(|event| serde_json::to_vec(&event.event).unwrap().len())
+ .sum();
+ let pause = HandoffPause {
+ intent: HandoffIntent {
+ handoff_id: "handoff".into(),
+ host_epoch: "host".into(),
+ root_run_id: "root".into(),
+ successor_run_id: "successor-run".into(),
+ successor_invocation_id: "successor-invocation".into(),
+ claim_id: "claim".into(),
+ },
+ remaining_steps: NonZeroU16::new(2).unwrap(),
+ execution: super::handoff::execution(),
+ };
+ let mut seal = RuntimeEvent::new(
+ source.invocation.clone(),
+ Fact::InvocationEnded {
+ outcome: InvocationOutcome::HandoffPaused {
+ pause: pause.clone(),
+ },
+ },
+ );
+ let mut widest = serde_json::to_value(&seal).unwrap();
+ widest["recorded_at"] =
+ serde_json::json!({ "secs_since_epoch": u64::MAX, "nanos_since_epoch": u32::MAX });
+ let mut reserved_bytes = serde_json::to_vec(&widest).unwrap().len();
+ let mut future = serde_json::to_value(&source).unwrap();
+ future["invocation"] =
+ serde_json::to_value(pause.intent.successor(&source.invocation)).unwrap();
+ future["recorded_at"] = widest["recorded_at"].clone();
+ future["fact"]["input"] = serde_json::json!({
+ "kind":"handoff", "pause": pause,
+ "claim": {
+ "id":pause.intent.claim_id,
+ "source":{"invocation":source.invocation,"high_water":u64::MAX,"digest":digest('a')},
+ "base":{"high_water":u64::MAX,"digest":digest('a')},
+ "replay":{"version":REPLAY_VERSION,"digest":digest('a'),"route_identity":digest('b')}
+ }
+ });
+ if reserve_successor {
+ reserved_bytes += serde_json::to_vec(&future).unwrap().len();
+ }
+ let mut completed = RuntimeEvent::new(
source.invocation.clone(),
- Fact::ModelRequested {
+ Fact::ModelCompleted {
step_id: "step".into(),
- model_id: "test".into(),
- purpose: Some(maka_runtime::context::ModelPurpose::Main),
- context: None,
- source_scope: context.source_evidence.scope,
- source_high_water: context.source_evidence.high_water,
- source_digest: context.source_evidence.digest,
- effective_source_digest: Some(context.effective_source_digest),
- input_digest: digest('a'),
- route_identity: digest('b'),
- checkpoint_event_id: None,
+ output: ModelStep {
+ parts: vec![ModelPart::Text {
+ text_kind: TextKind::Text,
+ text: String::new(),
+ provider_options: None,
+ }],
+ finish_reason: ModelFinishReason::Stop,
+ usage: ModelUsage::default(),
+ provider_options: None,
+ response_id: None,
+ model: None,
+ timestamp: None,
+ },
},
- ),
- )
- .await;
- let prefix = log
- .run_prefix("session", "root", None, 100, MAX_SOURCE_BYTES)
- .await
- .unwrap()
- .unwrap();
- let existing_bytes: usize = prefix
- .events
- .iter()
- .map(|event| serde_json::to_vec(&event.event).unwrap().len())
- .sum();
- let pause = HandoffPause {
- intent: HandoffIntent {
- handoff_id: "handoff".into(),
- host_epoch: "host".into(),
- root_run_id: "root".into(),
- successor_run_id: "successor-run".into(),
- successor_invocation_id: "successor-invocation".into(),
- claim_id: "claim".into(),
- },
- remaining_steps: NonZeroU16::new(2).unwrap(),
- };
- let mut seal = RuntimeEvent::new(
- source.invocation.clone(),
- Fact::InvocationEnded {
- outcome: InvocationOutcome::HandoffPaused {
- pause: pause.clone(),
+ );
+ let remaining = MAX_SOURCE_BYTES
+ - reserved_bytes
+ - existing_bytes
+ - serde_json::to_vec(&completed).unwrap().len();
+ let Fact::ModelCompleted { output, .. } = &mut completed.fact else {
+ unreachable!()
+ };
+ let ModelPart::Text { text, .. } = &mut output.parts[0] else {
+ unreachable!()
+ };
+ *text = "x".repeat(remaining);
+ append(&log, &completed).await;
+ if !reserve_successor {
+ assert!(
+ log.check_handoff(&source.invocation, &pause).await.is_err(),
+ "fitting the source seal alone does not make its successor readable"
+ );
+ assert!(log.append(&EventWrite::plain(seal).unwrap()).await.is_err());
+ log.close().await.unwrap();
+ continue;
+ }
+ log.check_handoff(&source.invocation, &pause).await.unwrap();
+ seal.recorded_at = UNIX_EPOCH + Duration::new(100_000_000_000, 999_999_999);
+ append(&log, &seal).await;
+ let sealed = log
+ .run_prefix("session", "root", None, MAX_SOURCE_EVENTS, MAX_SOURCE_BYTES)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(sealed.events.last().unwrap().event, seal);
+ assert_eq!(
+ sealed.high_water, 4,
+ "the pause remains claimable within the exact source budget"
+ );
+ let base = log
+ .context_before_run(&source.invocation, 100, MAX_SOURCE_BYTES)
+ .await
+ .unwrap();
+ let claim = ContinuationClaim {
+ id: pause.intent.claim_id.clone(),
+ source: RunBoundary {
+ invocation: sealed.invocation,
+ high_water: sealed.high_water,
+ digest: sealed.digest,
},
- },
- );
- let mut widest = serde_json::to_value(&seal).unwrap();
- widest["recorded_at"] =
- serde_json::json!({ "secs_since_epoch": u64::MAX, "nanos_since_epoch": u32::MAX });
- let reserved_bytes = serde_json::to_vec(&widest).unwrap().len();
- let mut completed = RuntimeEvent::new(
- source.invocation.clone(),
- Fact::ModelCompleted {
- step_id: "step".into(),
- output: ModelStep {
- parts: vec![ModelPart::Text {
- text_kind: TextKind::Text,
- text: String::new(),
- provider_options: None,
- }],
- finish_reason: ModelFinishReason::Stop,
- usage: ModelUsage::default(),
- provider_options: None,
- response_id: None,
- model: None,
- timestamp: None,
+ base: SessionBase {
+ high_water: base.source_evidence.high_water,
+ digest: base.source_evidence.digest,
+ },
+ replay: ReplayEvidence {
+ version: REPLAY_VERSION,
+ digest: digest('a'),
+ route_identity: digest('b'),
},
- },
- );
- let remaining = MAX_SOURCE_BYTES
- - reserved_bytes
- - existing_bytes
- - serde_json::to_vec(&completed).unwrap().len();
- let Fact::ModelCompleted { output, .. } = &mut completed.fact else {
- unreachable!()
- };
- let ModelPart::Text { text, .. } = &mut output.parts[0] else {
- unreachable!()
- };
- *text = "x".repeat(remaining);
- append(&log, &completed).await;
- log.check_handoff(&source.invocation, &pause).await.unwrap();
- seal.recorded_at = UNIX_EPOCH + Duration::new(100_000_000_000, 999_999_999);
- append(&log, &seal).await;
- let sealed = log
- .run_prefix("session", "root", None, MAX_SOURCE_EVENTS, MAX_SOURCE_BYTES)
+ };
+ let mut target = opening("unused", None);
+ target.invocation = pause.intent.successor(&source.invocation);
+ let Fact::InvocationOpened { input, .. } = &mut target.fact else {
+ unreachable!()
+ };
+ *input = InvocationInput::Handoff {
+ claim: Box::new(claim),
+ pause: Box::new(pause),
+ };
+ append(&log, &target).await;
+ log.read_model_context(
+ "session",
+ Some(&target.invocation.invocation_id),
+ MAX_SOURCE_EVENTS,
+ MAX_SOURCE_BYTES,
+ )
.await
- .unwrap()
.unwrap();
- assert_eq!(sealed.events.last().unwrap().event, seal);
- assert_eq!(
- sealed.high_water, 4,
- "the pause remains claimable within the exact source budget"
- );
- log.close().await.unwrap();
+ log.close().await.unwrap();
+ }
}
diff --git a/crates/event-log/tests/continuation/handoff_claim.rs b/crates/event-log/tests/continuation/handoff_claim.rs
index 783d93620d..978dc83e93 100644
--- a/crates/event-log/tests/continuation/handoff_claim.rs
+++ b/crates/event-log/tests/continuation/handoff_claim.rs
@@ -48,6 +48,7 @@ async fn handoff_claim_preserves_turn_and_frozen_ancestry_without_releasing_rese
claim_id: format!("claim-{round}"),
},
remaining_steps: NonZeroU16::new(3 - round).unwrap(),
+ execution: super::handoff::execution(),
};
append(
&log,
@@ -176,6 +177,31 @@ async fn handoff_claim_preserves_turn_and_frozen_ancestry_without_releasing_rese
let last = claim(&log, "manual", &source, base).await;
let manual = opening("manual-turn", Some(last));
append(&log, &manual).await;
+ let Fact::InvocationOpened {
+ input: InvocationInput::Continuation { claim, .. },
+ ..
+ } = &manual.fact
+ else {
+ unreachable!()
+ };
+ let mut pause = HandoffPause {
+ intent: HandoffIntent {
+ handoff_id: "manual-handoff".into(),
+ host_epoch: "host".into(),
+ root_run_id: manual.invocation.run_id.clone(),
+ successor_run_id: "manual-successor-run".into(),
+ successor_invocation_id: "manual-successor-invocation".into(),
+ claim_id: "manual-successor-claim".into(),
+ },
+ remaining_steps: NonZeroU16::new(2).unwrap(),
+ execution: super::handoff::execution(),
+ };
+ assert!(
+ log.check_handoff(&manual.invocation, &pause).await.is_err(),
+ "a physical handoff cannot discard manual resume's stable-cut policy"
+ );
+ pause.execution.replay_base = Some(claim.base.high_water);
+ log.check_handoff(&manual.invocation, &pause).await.unwrap();
close(&log, &manual).await;
let prefix = log
.run_prefix("session", &manual.invocation.run_id, None, 100, 65536)
@@ -245,6 +271,7 @@ async fn ineligible_handoff_never_seals_or_reserves_a_successor() {
claim_id: "claim".into(),
},
remaining_steps: NonZeroU16::new(3).unwrap(),
+ execution: super::handoff::execution(),
};
assert!(
log.check_handoff(&missing.invocation, &pause)
diff --git a/crates/presentation/tests/projection.rs b/crates/presentation/tests/projection.rs
index 561b8ea6a1..4d7c61b01d 100644
--- a/crates/presentation/tests/projection.rs
+++ b/crates/presentation/tests/projection.rs
@@ -172,6 +172,19 @@ fn completed_and_interrupted_rows_keep_overlay_identity_time_and_original_decode
claim_id: "claim".into(),
},
remaining_steps: std::num::NonZeroU16::new(2).unwrap(),
+ execution: Box::new(maka_runtime::handoff::HandoffExecution {
+ route_identity: format!("sha256:{}", "b".repeat(64)),
+ context: None,
+ provider_options: serde_json::json!({}),
+ main_output_limit: None,
+ supports_vision: false,
+ tools: maka_runtime::handoff::HandoffTools {
+ catalog_digest: format!("sha256:{}", "c".repeat(64)),
+ loaded: Default::default(),
+ },
+ compaction_attempted: false,
+ replay_base: None,
+ }),
},
},
] {
diff --git a/crates/runtime/src/handoff.rs b/crates/runtime/src/handoff.rs
index 44e0033ad6..f4a5d6e6b2 100644
--- a/crates/runtime/src/handoff.rs
+++ b/crates/runtime/src/handoff.rs
@@ -24,6 +24,9 @@ use crate::{event::Invocation, interaction::entity_id};
use serde::{Deserialize, Serialize};
use std::num::NonZeroU16;
+mod execution;
+pub use execution::{HandoffExecution, HandoffTools};
+
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct HandoffIntent {
@@ -41,6 +44,7 @@ pub struct HandoffPause {
pub intent: HandoffIntent,
/// Captured by the Engine, never restored from mutable Session policy.
pub remaining_steps: NonZeroU16,
+ pub execution: Box<HandoffExecution>,
}
impl HandoffIntent {
@@ -84,6 +88,11 @@ impl HandoffPause {
claim.validate_boundary(target)?;
self.validate(&claim.source.invocation)?;
if claim.id != self.intent.claim_id
+ || claim.replay.route_identity != self.execution.route_identity
+ || self
+ .execution
+ .replay_base
+ .is_some_and(|base| base != claim.base.high_water)
|| *target != self.intent.successor(&claim.source.invocation)
{
return Err("handoff must acquire its reserved successor in the same Turn");
@@ -93,6 +102,7 @@ impl HandoffPause {
pub fn validate(&self, source: &Invocation) -> Result<(), &'static str> {
self.intent.validate(source)?;
+ self.execution.validate()?;
if self.remaining_steps.get() > 256 {
return Err("handoff exceeds the Engine step budget");
}
diff --git a/crates/runtime/src/handoff/execution.rs b/crates/runtime/src/handoff/execution.rs
new file mode 100644
index 0000000000..db3a5adf4c
--- /dev/null
+++ b/crates/runtime/src/handoff/execution.rs
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::{archive::valid_projection_digest, context::ModelRequestContext};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::BTreeSet;
+
+/// Continuation settings, not credentials or process-local resources.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct HandoffExecution {
+ pub route_identity: String,
+ pub context: Option<ModelRequestContext>,
+ /// Provider-specific options intentionally retain their extensible JSON shape.
+ pub provider_options: Value,
+ pub main_output_limit: Option<u64>,
+ pub supports_vision: bool,
+ pub tools: HandoffTools,
+ pub compaction_attempted: bool,
+ /// Manual continuation's stable-cut policy, absent for ordinary conversation.
+ /// Physical handoff must not introduce or discard this projection policy.
+ pub replay_base: Option<u64>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct HandoffTools {
+ /// Covers definitions, nesting, execution semantics and discovery mode.
+ /// It does not promise persistence of a handler's executable implementation.
+ pub catalog_digest: String,
+ pub loaded: BTreeSet<String>,
+}
+
+impl HandoffExecution {
+ pub(super) fn validate(&self) -> Result<(), &'static str> {
+ if !valid_projection_digest(&self.route_identity)
+ || !valid_projection_digest(&self.tools.catalog_digest)
+ || self
+ .main_output_limit
+ .is_some_and(|n| n == 0 || n > 10_000_000_000)
+ || self.tools.loaded.len() > 128
+ || self
+ .tools
+ .loaded
+ .iter()
+ .any(|name| name.is_empty() || name.len() > 128)
+ {
+ return Err("invalid handoff execution settings");
+ }
+ if let Some(context) = &self.context {
+ context.validate()?;
+ }
+ Ok(())
+ }
+}
diff --git a/crates/tools/src/availability.rs b/crates/tools/src/availability.rs
index 32be66dcc4..94a6d4b07b 100644
--- a/crates/tools/src/availability.rs
+++ b/crates/tools/src/availability.rs
@@ -97,6 +97,30 @@ impl Availability {
pub fn clear(&self) {
self.active.lock().unwrap().clear();
}
+
+ pub fn checkpoint(&self) -> maka_runtime::handoff::HandoffTools {
+ maka_runtime::handoff::HandoffTools {
+ catalog_digest: self.catalog.digest(),
+ loaded: self.active.lock().unwrap().clone(),
+ }
+ }
+
+ pub fn restore(
+ &self,
+ checkpoint: &maka_runtime::handoff::HandoffTools,
+ ) -> Result<(), ToolError> {
+ if checkpoint.catalog_digest != self.catalog.digest()
+ || (!checkpoint.loaded.is_empty() && !self.enabled())
+ || checkpoint
+ .loaded
+ .iter()
+ .any(|name| direct(name) || !self.catalog.contains(name))
+ {
+ return Err(ToolError::Failed("handoff tool catalog changed".into()));
+ }
+ *self.active.lock().unwrap() = checkpoint.loaded.clone();
+ Ok(())
+ }
pub fn definition(&self) -> Option<ToolDefinition> {
if !self.enabled() {
return None;
diff --git a/crates/tools/src/catalog.rs b/crates/tools/src/catalog.rs
index ba5f759480..cef8b9674a 100644
--- a/crates/tools/src/catalog.rs
+++ b/crates/tools/src/catalog.rs
@@ -157,6 +157,25 @@ impl ToolCatalog {
}
}
+ /// Stable request-surface identity; process-local handler addresses are excluded.
+ pub fn digest(&self) -> String {
+ let entries: Vec<_> = self
+ .entries
+ .values()
+ .map(|entry| {
+ let r = &entry.registration;
+ (
+ &r.definition,
+ r.nesting == ToolNesting::Nestable,
+ r.semantics == ToolSemantics::ExclusiveStep,
+ )
+ })
+ .collect();
+ maka_runtime::artifact::content_digest(
+ &serde_json::to_vec(&(self.discovery, entries)).expect("tool definitions are JSON"),
+ )
+ }
+
pub fn semantics(&self, name: &str) -> Result<ToolSemantics, ToolRejection> {
self.entries
.get(name)
@@ -180,6 +199,10 @@ impl ToolCatalog {
self.entries.keys().cloned().collect()
}
+ pub(super) fn contains(&self, name: &str) -> bool {
+ self.entries.contains_key(name)
+ }
+
pub fn prepare(
&self,
name: String,
diff --git a/crates/tools/src/dispatch.rs b/crates/tools/src/dispatch.rs
index af184481a1..807b3672cd 100644
--- a/crates/tools/src/dispatch.rs
+++ b/crates/tools/src/dispatch.rs
@@ -61,6 +61,17 @@ impl RunTools {
self.availability.clear();
}
+ pub fn checkpoint(&self) -> maka_runtime::handoff::HandoffTools {
+ self.availability.checkpoint()
+ }
+
+ pub fn restore(
+ &self,
+ checkpoint: &maka_runtime::handoff::HandoffTools,
+ ) -> Result<(), ToolError> {
+ self.availability.restore(checkpoint)
+ }
+
/// Capture once per logical model step, before sending any physical attempt.
pub fn capture(&self) -> RequestTools<'_> {
let availability = if self.mode == ToolMode::CodeMode {
diff --git a/crates/tools/tests/integration/catalog.rs b/crates/tools/tests/integration/catalog.rs
index bf190a0475..dfafd3eadd 100644
--- a/crates/tools/tests/integration/catalog.rs
+++ b/crates/tools/tests/integration/catalog.rs
@@ -187,7 +187,7 @@ async fn discovery_reports_schema_limits_without_loading_blocked_tools_or_runnin
let run = RunTools::new(
log.clone(),
invocation.clone(),
- catalog,
+ catalog.clone(),
ToolMode::Direct,
CodeExecutor::new(1, CellLimits::default()).unwrap(),
);
@@ -224,6 +224,41 @@ async fn discovery_reports_schema_limits_without_loading_blocked_tools_or_runnin
);
assert!(result["blocked"]["schemaChars"].as_u64().unwrap() > 33_000);
let next = run.capture();
+ let checkpoint = run.checkpoint();
+ let restored = RunTools::new(
+ log.clone(),
+ invocation.clone(),
+ catalog.clone(),
+ ToolMode::Direct,
+ CodeExecutor::new(1, CellLimits::default()).unwrap(),
+ );
+ restored.restore(&checkpoint).unwrap();
+ assert_eq!(restored.capture().definitions(), next.definitions());
+ let mut invalid = checkpoint.clone();
+ invalid.loaded.insert("withheld".into());
+ assert!(restored.restore(&invalid).is_err());
+ assert_eq!(
+ restored.checkpoint(),
+ checkpoint,
+ "failed restore preserves the admitted view"
+ );
+ let changed = RunTools::new(
+ log.clone(),
+ invocation.clone(),
+ ToolCatalog::default(),
+ ToolMode::Direct,
+ CodeExecutor::new(1, CellLimits::default()).unwrap(),
+ );
+ assert!(changed.restore(&checkpoint).is_err());
+ drop(changed);
+ restored.clear_loaded();
+ assert!(restored.checkpoint().loaded.is_empty());
+ assert_eq!(
+ run.checkpoint(),
+ checkpoint,
+ "successor state does not share the old cache"
+ );
+ drop(restored);
run.clear_loaded();
assert_eq!(
next.definitions()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org