You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maka.apache.org by as...@apache.org on 2026/09/14 04:55:56 UTC
(maka) branch main updated: fix(runtime): unify safe stream recovery and preserve interrupted responses (#5270)
This is an automated email from the ASF dual-hosted git repository.
Astro-Han pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/maka.git
The following commit(s) were added to refs/heads/main by this push:
new f109ccde91 fix(runtime): unify safe stream recovery and preserve interrupted responses (#5270)
f109ccde91 is described below
commit f109ccde910dd8a42d35c6a72e60cef91815819f
Author: AstroHan <le...@outlook.com>
AuthorDate: Mon Sep 14 12:55:51 2026 +0800
fix(runtime): unify safe stream recovery and preserve interrupted responses (#5270)
Unify EOF, idle timeout, and retryable transport recovery under a ten-request budget while preserving failed response fragments as interrupted transcript output. Keep provider-side effects and continuation boundaries as replay barriers, and exclude failed fragments from model history.
Carry interruption markers through persistence and live/reload projection, render response dividers without empty thinking-only bubbles, and preserve finalized provider reasoning. Account for overflow recovery and sampled tool input, and remove redundant retry state.
Generated-by: Codex
---
apps/desktop/stories/app-shell.stories.tsx | 18 +-
packages/core/src/events.ts | 2 +
packages/core/src/runtime-event.ts | 4 +
packages/core/src/session.ts | 4 +-
.../session-continuity-coordinator.test.ts | 15 +-
.../src/__tests__/session-projector.test.ts | 7 +-
.../runtime-host/src/adapter/session-projector.ts | 10 +-
packages/runtime-host/src/protocol/index.ts | 3 +-
.../src/protocol/session-continuity.ts | 9 +
.../src/server/session-continuity-coordinator.ts | 5 +-
.../runtime/src/__tests__/ai-sdk-backend.test.ts | 530 ++++++++++++---------
.../runtime/src/__tests__/model-adapter.test.ts | 6 +-
.../__tests__/overflow-reactive-recovery.test.ts | 65 ++-
packages/runtime/src/ai-sdk-compaction.ts | 5 +-
packages/runtime/src/ai-sdk-message-projection.ts | 79 +--
packages/runtime/src/ai-sdk-turn.ts | 224 ++++-----
packages/runtime/src/model-adapter.ts | 2 +-
packages/runtime/src/model-protocol.ts | 4 +-
packages/runtime/src/runtime-event-backfill.ts | 8 +-
packages/runtime/src/runtime-event-read-model.ts | 1 +
.../runtime/src/session-event-runtime-mapper.ts | 3 +
.../src/__tests__/chat-turn-steering-order.test.ts | 18 +-
packages/ui/src/__tests__/materialize.test.ts | 22 +-
packages/ui/src/chat-turn.tsx | 16 +-
packages/ui/src/live-turn-projection.ts | 2 +
packages/ui/src/materialize.ts | 13 +-
packages/ui/src/timeline-fold.ts | 4 +
27 files changed, 634 insertions(+), 445 deletions(-)
diff --git a/apps/desktop/stories/app-shell.stories.tsx b/apps/desktop/stories/app-shell.stories.tsx
index 542c0551ad..043ac3f489 100644
--- a/apps/desktop/stories/app-shell.stories.tsx
+++ b/apps/desktop/stories/app-shell.stories.tsx
@@ -881,6 +881,7 @@ export const ProviderRetrying: Story = {
activeTurn: { turnId: 'turn-rr' },
messages: [
user('msg-rr-1', 'turn-rr', 1, '把这份长文档翻译成英文。'),
+ { ...assistant('failed-rr', 'turn-rr', 0, 'The project aims to improve the reliability of'), interrupted: true } as StoredMessage,
{ type: 'turn_state', id: 'state-rr', turnId: 'turn-rr', ts: NOW - 20_000, status: 'running' },
],
liveTurns: [{
@@ -894,10 +895,10 @@ export const ProviderRetrying: Story = {
turnId: 'turn-rr',
ts: NOW - 5_000,
attempt: 2,
- maxAttempts: 5,
+ maxAttempts: 10,
delayMs: 30_000,
remainingMs: 30_000,
- reason: 'rate_limit',
+ reason: 'stream_truncated',
},
receivedAtMs: NOW - 5_000,
},
@@ -912,6 +913,19 @@ export const ProviderRetrying: Story = {
},
};
+export const RecoveredResponse: Story = {
+ render: () => <ComposedShell chat={{ messages: [
+ user('retry-user', 'retry-turn', 1, '把这份长文档翻译成英文。'),
+ { ...assistant('retry-failed', 'retry-turn', 0, 'The project aims to improve the reliability of'), interrupted: true } as StoredMessage,
+ assistant('retry-success', 'retry-turn', 0, 'The project aims to improve the reliability of model responses and preserve completed work when a connection is interrupted.'),
+ { type: 'turn_state', id: 'retry-complete', turnId: 'retry-turn', ts: NOW, status: 'completed' },
+ ] }} />,
+ play: async ({ canvasElement }) => {
+ await waitFor(() => expect(canvasElement.querySelectorAll('[data-response-interrupted="true"]')).toHaveLength(1));
+ expect(canvasElement.textContent).toContain('preserve completed work');
+ },
+};
+
// Real path: the app restarted mid-turn, so the last turn is failed with
// errorClass 'app_restarted' and offers safe-resume. The warning-severity
// Banner carries the 继续这一轮 button (`safeResumeAction`) — the recovery
diff --git a/packages/core/src/events.ts b/packages/core/src/events.ts
index d103137354..91b40e889b 100644
--- a/packages/core/src/events.ts
+++ b/packages/core/src/events.ts
@@ -617,6 +617,7 @@ export interface TextDeltaEvent extends BaseEvent {
export interface TextCompleteEvent extends BaseEvent {
type: 'text_complete';
+ interrupted?: true;
messageId: string;
text: string;
/** Provider-owned text metadata such as Responses URL citations. */
@@ -633,6 +634,7 @@ export interface ThinkingDeltaEvent extends BaseEvent {
export interface ThinkingCompleteEvent extends BaseEvent {
type: 'thinking_complete';
+ interrupted?: true;
messageId: string;
text: string;
/** Anthropic signed thinking — MUST be re-sent on replay. */
diff --git a/packages/core/src/runtime-event.ts b/packages/core/src/runtime-event.ts
index 052bd78f8b..635b5257fe 100644
--- a/packages/core/src/runtime-event.ts
+++ b/packages/core/src/runtime-event.ts
@@ -182,6 +182,8 @@ export function isTerminalRuntimeEventStatus(value: unknown): boolean {
export interface RuntimeEventTextContent extends MessageContent {
kind: 'text';
+ /** Failed response fragment retained for display, never for model replay. */
+ interrupted?: true;
/** Provider-owned text metadata such as Responses URL citations. */
providerOptions?: Record<string, unknown>;
/** Durable provenance for a host-authored user-role turn. */
@@ -724,6 +726,7 @@ const TEXT_CONTENT_SHAPE = defineObjectShape<RuntimeEventTextContent>()(
['kind', 'text'],
[
'displayText',
+ 'interrupted',
'origin',
'attachments',
'directoryReferences',
@@ -1046,6 +1049,7 @@ function isRuntimeEventContent(value: unknown): value is RuntimeEventContent {
!hasExactShape(value, TEXT_CONTENT_SHAPE) ||
(value.origin !== undefined && !isTurnOrigin(value.origin)) ||
(value.steering !== undefined && value.steering !== true) ||
+ (value.interrupted !== undefined && value.interrupted !== true) ||
(value.providerOptions !== undefined && !isRecord(value.providerOptions))
) {
return false;
diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts
index 097f9bfbbd..76bcc27a4c 100644
--- a/packages/core/src/session.ts
+++ b/packages/core/src/session.ts
@@ -800,6 +800,7 @@ export function isUserVisibleSessionSystemNote(kind: string): boolean {
export interface AssistantMessage {
type: 'assistant';
+ interrupted?: true;
id: string;
turnId: string;
ts: number;
@@ -1256,7 +1257,7 @@ const USER_MESSAGE_SHAPE = defineObjectShape<UserMessage>()(
);
const ASSISTANT_MESSAGE_SHAPE = defineObjectShape<AssistantMessage>()(
['type', 'id', 'turnId', 'ts', 'text', 'modelId'],
- ['thinking', 'contentOrder', 'providerOptions'],
+ ['thinking', 'contentOrder', 'providerOptions', 'interrupted'],
);
const TOOL_CALL_MESSAGE_SHAPE = defineObjectShape<ToolCallMessage>()(
['type', 'id', 'turnId', 'ts', 'toolName', 'args'],
@@ -1548,6 +1549,7 @@ function decodeMessage(
hasMessageEnvelope(message, true) &&
typeof message.text === 'string' &&
typeof message.modelId === 'string' &&
+ (message.interrupted === undefined || message.interrupted === true) &&
(message.providerOptions === undefined || isRecord(message.providerOptions)) &&
(message.thinking === undefined || isAssistantThinking(message.thinking)) &&
(message.contentOrder === undefined ||
diff --git a/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts b/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts
index 42bf3a14bf..74b0109e90 100644
--- a/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts
+++ b/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts
@@ -361,12 +361,17 @@ test('publishes a non-prefix final value as an authoritative replacement', async
...textEvent(1),
text: 'draft',
});
- await coordinator.acceptRuntimeEvent(
- SESSION_ID,
- 'run-1',
- textCompleteEvent('message-1', 'final'),
- );
+ await coordinator.acceptRuntimeEvent(SESSION_ID, 'run-1', {
+ ...textCompleteEvent('message-1', 'final'),
+ interrupted: true,
+ });
await waitFor(() => sink.frames.length === 3);
+ const interrupted = decodeSubscriptionFrame(
+ JSON.parse(encodeProtocolMessage(sink.frames[2]!).toString('utf8')),
+ );
+ assert.ok(
+ interrupted.kind === 'subscription.session_delta' && interrupted.delta.interrupted === true,
+ );
assert.deepEqual(
sink.frames.map((frame) =>
diff --git a/packages/runtime-host/src/__tests__/session-projector.test.ts b/packages/runtime-host/src/__tests__/session-projector.test.ts
index 9e50f6fc25..129efa5359 100644
--- a/packages/runtime-host/src/__tests__/session-projector.test.ts
+++ b/packages/runtime-host/src/__tests__/session-projector.test.ts
@@ -88,7 +88,10 @@ test('applies authoritative replacement once and does not complete it again at T
['text_delta'],
);
assert.deepEqual(projector.accept(deltaFrame(1, 0, 'final', { reset: true })).events, []);
- const completed = projector.accept(deltaFrame(2, 5, '', { complete: true })).events;
+ const completed = projector.accept(
+ deltaFrame(2, 5, '', { complete: true, interrupted: true }),
+ ).events;
+ assert.ok(completed[0]?.type === 'text_complete' && completed[0].interrupted === true);
assert.deepEqual(
completed.map((event) => [event.type, 'text' in event ? event.text : '']),
[['text_complete', 'final']],
@@ -916,7 +919,7 @@ function deltaFrame(
sequence: number,
startOffset: number,
text: string,
- flags: { reset?: true; complete?: true } = {},
+ flags: { reset?: true; complete?: true; interrupted?: true } = {},
): SubscriptionFrame {
return {
kind: 'subscription.session_delta',
diff --git a/packages/runtime-host/src/adapter/session-projector.ts b/packages/runtime-host/src/adapter/session-projector.ts
index 548b920d3b..944644e1c2 100644
--- a/packages/runtime-host/src/adapter/session-projector.ts
+++ b/packages/runtime-host/src/adapter/session-projector.ts
@@ -38,6 +38,7 @@ import type {
} from '../protocol/index.js';
interface AssistantAccumulator {
+ interrupted?: true;
kind: 'text' | 'thinking';
turnId: string;
messageId: string;
@@ -125,12 +126,13 @@ export class RuntimeHostSessionProjector {
replacing: false,
});
}
- if (message.text) {
+ if (message.text || message.interrupted) {
this.#accumulators.set(accumulatorKey('text', message.id), {
kind: 'text',
turnId: root.turnId,
messageId: message.id,
text: message.text,
+ ...(message.interrupted ? { interrupted: true } : {}),
complete: true,
replacing: false,
});
@@ -271,7 +273,7 @@ export class RuntimeHostSessionProjector {
text: message.thinking.text,
});
}
- if (message.text) {
+ if (message.text || message.interrupted) {
events.push({
type: 'text_complete',
id: `${terminal.id}:text:${message.id}`,
@@ -279,6 +281,7 @@ export class RuntimeHostSessionProjector {
messageId: message.id,
ts: terminal.ts,
text: message.text,
+ ...(message.interrupted ? { interrupted: true } : {}),
});
}
}
@@ -366,6 +369,7 @@ export class RuntimeHostSessionProjector {
turnId: delta.turnId,
messageId: delta.messageId,
text: folded.text,
+ ...(delta.interrupted ? { interrupted: true } : {}),
complete: delta.complete === true,
replacing: delta.complete === true ? false : replacing,
});
@@ -377,6 +381,7 @@ export class RuntimeHostSessionProjector {
messageId: delta.messageId,
ts: this.#now(),
text: folded.text,
+ ...(delta.interrupted ? { interrupted: true } : {}),
});
} else if (folded.tail && !replacing) {
events.push({
@@ -484,6 +489,7 @@ export class RuntimeHostSessionProjector {
messageId: accumulator.messageId,
ts: this.#now(),
text: accumulator.text,
+ ...(accumulator.interrupted ? { interrupted: true } : {}),
});
}
if (root.status === 'completed') {
diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts
index eeea526f0e..3372e7bd30 100644
--- a/packages/runtime-host/src/protocol/index.ts
+++ b/packages/runtime-host/src/protocol/index.ts
@@ -101,7 +101,8 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const;
export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const;
// Increment when the same protocol version no longer guarantees safe Client-Host
// interoperability. Mismatches are rejected before domain commands are admitted.
-export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 151 as const;
+export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 152 as const;
+// 152: Assistant completions and transcript rows preserve interrupted responses.
// 151: WorkHub selects and delegates through a durable Host Form interaction.
// 150: Message admission accepts an empty-text Message that carries a quote or
// an attachment (#4804). Peers older than this epoch reject that frame at
diff --git a/packages/runtime-host/src/protocol/session-continuity.ts b/packages/runtime-host/src/protocol/session-continuity.ts
index 318e07c851..490e078433 100644
--- a/packages/runtime-host/src/protocol/session-continuity.ts
+++ b/packages/runtime-host/src/protocol/session-continuity.ts
@@ -142,6 +142,7 @@ export interface SessionProjectionFrame extends SubscriptionEnvelope {
}
export interface SessionAssistantDelta {
+ interrupted?: true;
kind: 'text' | 'thinking';
turnId: string;
runId: string;
@@ -739,6 +740,7 @@ function decodeAssistantDelta(value: unknown): SessionAssistantDelta {
'text',
'reset',
'complete',
+ 'interrupted',
]);
assertRequiredKeys(record, 'Session assistant delta', [
'kind',
@@ -754,6 +756,12 @@ function decodeAssistantDelta(value: unknown): SessionAssistantDelta {
if (record.complete !== undefined && record.complete !== true) {
throw invalidProtocolFrame('Invalid Session assistant delta completion');
}
+ if (
+ record.interrupted !== undefined &&
+ (record.interrupted !== true || record.complete !== true)
+ ) {
+ throw invalidProtocolFrame('Interrupted assistant delta must be complete');
+ }
if (record.reset !== undefined && record.reset !== true) {
throw invalidProtocolFrame('Invalid Session assistant delta reset');
}
@@ -779,6 +787,7 @@ function decodeAssistantDelta(value: unknown): SessionAssistantDelta {
),
...(record.reset === true ? { reset: true as const } : {}),
...(record.complete === true ? { complete: true as const } : {}),
+ ...(record.interrupted === true ? { interrupted: true as const } : {}),
};
}
diff --git a/packages/runtime-host/src/server/session-continuity-coordinator.ts b/packages/runtime-host/src/server/session-continuity-coordinator.ts
index e6f9d3fbbc..1e1c8030a4 100644
--- a/packages/runtime-host/src/server/session-continuity-coordinator.ts
+++ b/packages/runtime-host/src/server/session-continuity-coordinator.ts
@@ -775,7 +775,7 @@ export class SessionContinuityCoordinator implements SessionContinuityService {
const textKey = assistantStreamKey('text', event.messageId);
const text = state.assistantStreams.get(textKey);
- if (text || event.text.length > 0) {
+ if (text || event.text.length > 0 || event.interrupted) {
const current =
text ??
({
@@ -792,6 +792,7 @@ export class SessionContinuityCoordinator implements SessionContinuityService {
current,
'text',
event.text,
+ event.interrupted,
);
}
state.assistantStreams.delete(textKey);
@@ -1631,6 +1632,7 @@ export class SessionContinuityCoordinator implements SessionContinuityService {
current: Pick<ActiveAssistantStream, 'turnId' | 'messageId' | 'text'>,
kind: SessionAssistantDelta['kind'],
finalText: string,
+ interrupted?: true,
): void {
const extendsPrefix = finalText.startsWith(current.text);
const suffix = extendsPrefix ? finalText.slice(current.text.length) : finalText;
@@ -1662,6 +1664,7 @@ export class SessionContinuityCoordinator implements SessionContinuityService {
text: '',
...(!extendsPrefix && finalText.length === 0 ? { reset: true as const } : {}),
complete: true,
+ ...(interrupted ? { interrupted: true } : {}),
},
});
}
diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts
index 9cdf8b4afc..4b19e3308a 100644
--- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts
+++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts
@@ -75,6 +75,7 @@ import {
} from '../history-compact-checkpoint.js';
import { buildDefaultContextBudgetPolicy } from '../context-budget-policy.js';
import { buildRuntimeEventModelReplayPlan, buildSteeringEnvelope } from '../model-history.js';
+import { backfillRuntimeEventsFromStoredMessages } from '../runtime-event-backfill.js';
import { HistoryCompactSummarizerError } from '../history-compact-summarizer.js';
import { SandboxCommandError } from '../sandbox/errors.js';
import { buildRequestSandboxBoundaryTool } from '../sandbox-boundary-tool.js';
@@ -8196,8 +8197,8 @@ describe('AiSdkBackend usage telemetry', () => {
reason,
})),
[
- { phase: 'scheduled', attempt: 2, maxAttempts: 2, reason: 'stream_truncated' },
- { phase: 'started', attempt: 2, maxAttempts: 2, reason: 'stream_truncated' },
+ { phase: 'scheduled', attempt: 2, maxAttempts: 10, reason: 'stream_truncated' },
+ { phase: 'started', attempt: 2, maxAttempts: 10, reason: 'stream_truncated' },
],
);
assert.equal(
@@ -8207,6 +8208,104 @@ describe('AiSdkBackend usage telemetry', () => {
assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'end_turn');
});
+ for (const visibleText of [false, true]) {
+ test(`isolates undispatched local calls after truncation (visible text: ${visibleText})`, async () => {
+ const durable = durableTurnHarness('turn-local-truncated', 'do the work');
+ const executed: string[] = [];
+ const assistants: AssistantMessage[] = [];
+ let calls = 0;
+ const model = new MockLanguageModelV4({
+ doStream: async () => {
+ calls += 1;
+ const chunks: LanguageModelV4StreamPart[] = [{ type: 'stream-start', warnings: [] }];
+ if (calls === 2 && visibleText) {
+ chunks.push(
+ { type: 'text-start', id: 'partial' },
+ { type: 'text-delta', id: 'partial', delta: 'Partial answer' },
+ );
+ }
+ if (calls < 4) {
+ chunks.push({
+ type: 'tool-call',
+ toolCallId: `call-${calls}`,
+ toolName: 'Write',
+ input: JSON.stringify({ value: ['prior', 'discarded', 'fresh'][calls - 1] }),
+ providerExecuted: false,
+ });
+ } else {
+ chunks.push(
+ { type: 'text-start', id: 'final' },
+ { type: 'text-delta', id: 'final', delta: 'Done' },
+ { type: 'text-end', id: 'final' },
+ );
+ }
+ if (calls !== 2) {
+ chunks.push({
+ type: 'finish',
+ finishReason: { unified: calls < 4 ? 'tool-calls' : 'stop', raw: 'stop' },
+ usage: {
+ inputTokens: { total: 7, noCache: 7, cacheRead: 0, cacheWrite: 0 },
+ outputTokens: { total: 3, text: 3, reasoning: 0 },
+ },
+ });
+ }
+ return {
+ stream: simulateReadableStream({
+ chunks,
+ initialDelayInMs: null,
+ chunkDelayInMs: null,
+ }),
+ };
+ },
+ });
+ const backend = createBackend({
+ connection: connection(),
+ modelId: 'mock-model-id',
+ modelFactory: () => model,
+ tools: [
+ {
+ ...testTool('Write', z.object({ value: z.string() })),
+ impl: async (input) => {
+ executed.push((input as { value: string }).value);
+ return { ok: true };
+ },
+ },
+ ],
+ appendMessage: async (message) => {
+ if (message.type === 'assistant') assistants.push(message);
+ },
+ loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents,
+ providerRetrySleep: async () => {},
+ });
+ const events = await drainDurably(backend.send(durable.input()), durable);
+ assert.equal(calls, 4);
+ assert.deepEqual(executed, ['prior', 'fresh']);
+ assert.equal(
+ events.some(
+ (event) =>
+ (event.type === 'tool_start' || event.type === 'tool_result') &&
+ event.toolUseId === 'call-2',
+ ),
+ false,
+ );
+ assert.equal(JSON.stringify(durable.ledger).includes('discarded'), false);
+ assert.deepEqual(
+ assistants.map((message) => message.text),
+ visibleText ? ['Partial answer', 'Done'] : ['Done'],
+ );
+ const error = events.find((event) => event.type === 'error');
+ assert.equal(error, undefined);
+ if (visibleText) assert.equal(assistants[0]?.interrupted, true);
+ assert.equal(
+ events.some((event) => event.type === 'token_usage'),
+ false,
+ );
+ assert.equal(JSON.stringify(model.doStreamCalls.slice(2)).includes('discarded'), false);
+ assert.equal(JSON.stringify(model.doStreamCalls.slice(2)).includes('Partial answer'), false);
+ assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'end_turn');
+ });
+ }
+
test('records exhaustion of output-free truncated stream recovery', async () => {
const durable = durableTurnHarness('turn-truncated-exhausted', 'analyse the image');
let calls = 0;
@@ -8243,94 +8342,51 @@ describe('AiSdkBackend usage telemetry', () => {
(event): event is Extract<SessionEvent, { type: 'error' }> => event.type === 'error',
);
- assert.equal(calls, 2);
+ assert.equal(calls, 10);
assert.equal(error?.reason, 'stream_truncated');
- assert.deepEqual(error?.retry, { decision: 'exhausted', attempts: 2 });
+ assert.deepEqual(error?.retry, { decision: 'exhausted', attempts: 10 });
assert.equal(error?.message, 'Provider stream ended without finishing (other)');
assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'error');
});
- for (const output of ['text', 'tool'] as const) {
- // The upstream cut the SSE connection mid-answer: chunks arrived, no
- // `finish` frame did. The stream then ends without yielding an error and
- // without throwing, so every guard that watches for a thrown failure sees
- // nothing. Reporting `end_turn` here tells the caller the model said its
- // piece when the connection simply died — a benchmark cell recorded
- // `status: completed` on exactly this shape while the agent was still
- // mid-task.
- test(`does not retry a truncated provider stream after ${output} activity`, async () => {
- const durable = durableTurnHarness('turn-truncated', 'analyse the image');
- let calls = 0;
- const model = new MockLanguageModelV4({
- doStream: async () => {
- calls += 1;
- return {
- stream: simulateReadableStream({
- chunks: [
- { type: 'stream-start', warnings: [] },
- ...(output === 'text'
- ? [
- { type: 'text-start', id: 'text-1' },
- { type: 'text-delta', id: 'text-1', delta: 'Let me look at the top region' },
- ]
- : [
- {
- type: 'tool-input-start',
- id: 'search-1',
- toolName: 'web_search',
- providerExecuted: true,
- },
- ]),
- ] as LanguageModelV4StreamPart[],
- initialDelayInMs: null,
- chunkDelayInMs: null,
- }),
- };
- },
- });
- const backend = createBackend({
- connection: connection(),
- modelId: 'mock-model-id',
- modelFactory: () => model,
- tools: [],
- loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents,
- providerRetrySleep: async () => {},
- });
-
- const events = await drainDurably(backend.send(durable.input()), durable);
- const complete = events.find(
- (event): event is Extract<SessionEvent, { type: 'complete' }> => event.type === 'complete',
- );
-
- // Not merely "some other stop reason": `max_tokens` would also satisfy that
- // and still record the turn as completed downstream, which is the bug.
- assert.equal(
- complete?.stopReason,
- 'error',
- 'a stream that never delivered a finish frame did not end the turn',
- );
- assert.equal(calls, 1);
- assert.equal(
- events.some((event) => event.type === 'provider_retry'),
- false,
- );
- // And it must say so. A failed terminal whose only trace is the stop reason
- // leaves the session's lastError empty and the request ledger reading
- // `success` — the same silence that let the benchmark cell pass unnoticed.
- assert.ok(
- events.some((event) => event.type === 'error'),
- 'a failed terminal must be accompanied by an error event',
- );
- const error = events.find(
- (event): event is Extract<SessionEvent, { type: 'error' }> => event.type === 'error',
- );
- assert.equal(error?.reason, 'stream_truncated', error?.message ?? 'error event missing');
- assert.deepEqual(error?.retry, {
- decision: 'declined',
- because: output === 'tool' ? 'side_effects' : 'observable_output',
- });
+ test('does not retry a truncated stream after provider tool input starts', async () => {
+ const durable = durableTurnHarness('turn-truncated', 'analyse the image');
+ let calls = 0;
+ const model = new MockLanguageModelV4({
+ doStream: async () => {
+ calls += 1;
+ return {
+ stream: simulateReadableStream({
+ chunks: [
+ { type: 'stream-start', warnings: [] },
+ {
+ type: 'tool-input-start',
+ id: 'search-1',
+ toolName: 'web_search',
+ providerExecuted: true,
+ },
+ ] as LanguageModelV4StreamPart[],
+ initialDelayInMs: null,
+ chunkDelayInMs: null,
+ }),
+ };
+ },
});
- }
+ const backend = createBackend({
+ connection: connection(),
+ modelId: 'mock-model-id',
+ modelFactory: () => model,
+ tools: [],
+ loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents,
+ providerRetrySleep: async () => {},
+ });
+ const events = await drainDurably(backend.send(durable.input()), durable);
+ assert.equal(calls, 1);
+ const error = events.find((event) => event.type === 'error');
+ assert.equal(error?.reason, 'stream_truncated');
+ assert.deepEqual(error?.retry, { decision: 'declined', because: 'side_effects' });
+ assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'error');
+ });
test('rejects continuation-capable tools before side effects without a durable reader', async () => {
const loop = countingToolLoopModel(1);
@@ -10398,8 +10454,8 @@ describe('AiSdkBackend RunTrace', () => {
reason,
})),
[
- { phase: 'scheduled', attempt: 2, maxAttempts: 2, reason: 'timeout' },
- { phase: 'started', attempt: 2, maxAttempts: 2, reason: 'timeout' },
+ { phase: 'scheduled', attempt: 2, maxAttempts: 10, reason: 'timeout' },
+ { phase: 'started', attempt: 2, maxAttempts: 10, reason: 'timeout' },
],
);
assert.equal(
@@ -10414,7 +10470,7 @@ describe('AiSdkBackend RunTrace', () => {
assert.notEqual(assistants[0]?.id, assistants[1]?.id);
});
- test('retries a retryable network failure after partial thinking by sealing it', async () => {
+ test('seals partial thinking and discards local tool intents after a retryable network failure', async () => {
// Incident shape: the provider streamed thinking deltas, then the
// connection reset mid-step (ECONNRESET after ~120s). Recovery safety
// depends on what the attempt emitted, not on which side detected the
@@ -10452,6 +10508,13 @@ describe('AiSdkBackend RunTrace', () => {
{ type: 'stream-start', warnings: [] },
{ type: 'reasoning-start', id: 'reasoning-1' },
{ type: 'reasoning-delta', id: 'reasoning-1', delta: 'partial thought' },
+ {
+ type: 'tool-call',
+ toolCallId: 'discarded-call',
+ toolName: 'Write',
+ input: '{"value":"discarded"}',
+ providerExecuted: false,
+ },
],
connectionResetFailure(),
);
@@ -10466,7 +10529,7 @@ describe('AiSdkBackend RunTrace', () => {
connection: connection(),
modelId: 'mock-model-id',
modelFactory: () => model,
- tools: [],
+ tools: [testTool('Write', z.object({ value: z.string() }))],
loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents,
providerRetrySleep: async () => {},
});
@@ -10491,8 +10554,8 @@ describe('AiSdkBackend RunTrace', () => {
reason,
})),
[
- { phase: 'scheduled', attempt: 2, maxAttempts: 2, reason: 'network' },
- { phase: 'started', attempt: 2, maxAttempts: 2, reason: 'network' },
+ { phase: 'scheduled', attempt: 2, maxAttempts: 10, reason: 'network' },
+ { phase: 'started', attempt: 2, maxAttempts: 10, reason: 'network' },
],
);
assert.equal(
@@ -10511,6 +10574,11 @@ describe('AiSdkBackend RunTrace', () => {
const retryPrompt = JSON.stringify(model.doStreamCalls[1]?.prompt);
assert.equal(retryPrompt.includes('partial thought'), false);
assert.match(retryPrompt, /review the commits/);
+ assert.equal(
+ events.some((event) => event.type === 'tool_start' || event.type === 'tool_result'),
+ false,
+ );
+ assert.equal(JSON.stringify(durable.ledger).includes('discarded-call'), false);
});
test('retries a retryable network failure before any observable output', async () => {
@@ -10572,11 +10640,7 @@ describe('AiSdkBackend RunTrace', () => {
assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'end_turn');
});
- test('stops after one sealed-thinking network recovery in the same provider step', async () => {
- // Every attempt streams thinking and is cut mid-stream. The first cut
- // seals and retries; the second is terminal, so one recovery per step
- // bounds how many severed-thinking fragments a systematically cutting
- // gateway can leave in the transcript.
+ test('bounds repeated thinking failures by the shared request budget', async () => {
const durable = durableTurnHarness('turn-econnreset-thinking-budget', 'review the commits');
const assistants: AssistantMessage[] = [];
let failCurrentStream: (() => void) | undefined;
@@ -10621,92 +10685,111 @@ describe('AiSdkBackend RunTrace', () => {
}
}
- assert.equal(calls, 2);
+ assert.equal(calls, 10);
assert.equal(
events.filter(
(event): event is Extract<SessionEvent, { type: 'provider_retry' }> =>
event.type === 'provider_retry' && event.phase === 'scheduled',
).length,
- 1,
+ 9,
);
const error = events.find(
(event): event is Extract<SessionEvent, { type: 'error' }> => event.type === 'error',
);
assert.equal(error?.reason, 'network');
assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'error');
- assert.equal(assistants.length, 2);
+ assert.equal(assistants.length, 10);
assert.equal(assistants[0]?.thinking?.text, 'partial thought 1');
assert.equal(assistants[1]?.thinking?.text, 'partial thought 2');
assert.notEqual(assistants[0]?.id, assistants[1]?.id);
});
- test('does not retry a network failure after provider continuation metadata on thinking', async () => {
- // Continuation identity (Responses reasoning item ids, encrypted
- // content) cannot be replayed into a fresh request, so thinking that
- // carries it stays non-recoverable even though the failure itself is
- // retryable. The second reasoning part's delta is the fail trigger:
- // stream ordering guarantees the metadata on the first part's
- // reasoning-end was already consumed when it arrives.
- const durable = durableTurnHarness('turn-econnreset-metadata', 'review the commits');
- let failCurrentStream: (() => void) | undefined;
- let calls = 0;
- const model = new MockLanguageModelV4({
- doStream: async () => {
- calls += 1;
- const failing = midStreamFailureStream(
- [
- { type: 'stream-start', warnings: [] },
- { type: 'reasoning-start', id: 'reasoning-1' },
- {
- type: 'reasoning-delta',
- id: 'reasoning-1',
- delta: 'completed provider reasoning',
- },
- {
- type: 'reasoning-end',
- id: 'reasoning-1',
- providerMetadata: {
- openai: {
- itemId: 'reasoning-item-1',
- reasoningEncryptedContent: 'encrypted-reasoning',
- },
- },
- },
- { type: 'reasoning-start', id: 'reasoning-2' },
- { type: 'reasoning-delta', id: 'reasoning-2', delta: 'second thought' },
- ],
- connectionResetFailure(),
- );
- failCurrentStream = failing.fail;
- return { stream: failing.stream };
+ for (const { label, providerMetadata } of [
+ {
+ label: 'encrypted Responses',
+ providerMetadata: {
+ openai: { itemId: 'reasoning-item-1', reasoningEncryptedContent: 'encrypted-reasoning' },
},
- });
- const backend = createBackend({
- connection: connection(),
- modelId: 'mock-model-id',
- modelFactory: () => model,
- tools: [],
- loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents,
- providerRetrySleep: async () => {},
- });
+ },
+ {
+ label: 'redacted Anthropic',
+ providerMetadata: { anthropic: { redactedData: 'redacted-reasoning' } },
+ },
+ ] as { label: string; providerMetadata: Record<string, Record<string, string>> }[]) {
+ test(`preserves finalized ${label} thinking when the next part fails without retrying`, async () => {
+ // Continuation identity (Responses reasoning item ids, encrypted
+ // content) cannot be replayed into a fresh request, so thinking that
+ // carries it stays non-recoverable even though the failure itself is
+ // retryable. The second reasoning part's delta is the fail trigger:
+ // stream ordering guarantees the metadata on the first part's
+ // reasoning-end was already consumed when it arrives.
+ const durable = durableTurnHarness('turn-econnreset-metadata', 'review the commits');
+ let failCurrentStream: (() => void) | undefined;
+ let calls = 0;
+ const model = new MockLanguageModelV4({
+ doStream: async () => {
+ calls += 1;
+ const failing = midStreamFailureStream(
+ [
+ { type: 'stream-start', warnings: [] },
+ { type: 'reasoning-start', id: 'reasoning-1', providerMetadata },
+ {
+ type: 'reasoning-delta',
+ id: 'reasoning-1',
+ delta: 'completed provider reasoning',
+ },
+ {
+ type: 'reasoning-end',
+ id: 'reasoning-1',
+ providerMetadata,
+ },
+ { type: 'reasoning-start', id: 'reasoning-2' },
+ { type: 'reasoning-delta', id: 'reasoning-2', delta: 'second thought' },
+ ],
+ connectionResetFailure(),
+ );
+ failCurrentStream = failing.fail;
+ return { stream: failing.stream };
+ },
+ });
+ const backend = createBackend({
+ connection: connection(),
+ modelId: 'mock-model-id',
+ modelFactory: () => model,
+ tools: [],
+ loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents,
+ providerRetrySleep: async () => {},
+ });
- const events: SessionEvent[] = [];
- for await (const event of backend.send(durable.input())) {
- durable.record(event);
- events.push(event);
- if (event.type === 'thinking_delta' && event.text === 'second thought') {
- failCurrentStream?.();
+ const events: SessionEvent[] = [];
+ for await (const event of backend.send(durable.input())) {
+ durable.record(event);
+ events.push(event);
+ if (event.type === 'thinking_delta' && event.text === 'second thought') {
+ failCurrentStream?.();
+ }
}
- }
- assert.equal(calls, 1);
- assert.equal(
- events.some((event) => event.type === 'provider_retry'),
- false,
- );
- assert.equal(events.find((event) => event.type === 'error')?.reason, 'network');
- assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'error');
- });
+ assert.equal(calls, 1);
+ assert.equal(
+ events.some((event) => event.type === 'provider_retry'),
+ false,
+ );
+ assert.equal(events.find((event) => event.type === 'error')?.reason, 'network');
+ assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'error');
+ assert.deepEqual(
+ durable.ledger.flatMap((event) =>
+ event.content?.kind === 'thinking'
+ ? [[event.content.text, event.modelVisibility ?? 'visible']]
+ : [],
+ ),
+ [
+ ['completed provider reasoning', 'visible'],
+ ['second thought', 'hidden'],
+ ],
+ );
+ });
+ }
test('retries DeepSeek OpenAI Chat reasoning marked only for field replay', async () => {
const timers = manualWatchdogTimer();
@@ -10893,56 +10976,7 @@ describe('AiSdkBackend RunTrace', () => {
assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'end_turn');
});
- test('stops after one recovered idle watchdog timeout in the same provider step', async () => {
- const timers = manualWatchdogTimer();
- let calls = 0;
- const model = new MockLanguageModelV4({
- doStream: async (options) => {
- calls += 1;
- return {
- stream: hangingProviderStream(
- [
- { type: 'stream-start', warnings: [] },
- { type: 'reasoning-start', id: `reasoning-${calls}` },
- {
- type: 'reasoning-delta',
- id: `reasoning-${calls}`,
- delta: `partial thought ${calls}`,
- },
- ],
- options.abortSignal,
- ),
- };
- },
- });
- const backend = createBackend({
- connection: connection(),
- modelId: 'mock-model-id',
- modelFactory: () => model,
- tools: [],
- streamWatchdogTimer: timers.clock,
- providerRetrySleep: async () => {},
- });
-
- const events: SessionEvent[] = [];
- for await (const event of backend.send({ turnId: 'turn-1', text: 'hi', context: [] })) {
- events.push(event);
- if (event.type === 'thinking_delta' && event.text.startsWith('partial thought')) {
- timers.fire();
- }
- }
-
- assert.equal(calls, 2);
- assert.equal(
- events.filter((event) => event.type === 'provider_retry' && event.phase === 'scheduled')
- .length,
- 1,
- );
- assert.equal(events.find((event) => event.type === 'error')?.reason, 'timeout');
- assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'error');
- });
-
- test('retries post-tool continuation once without re-running the durable tool result', async () => {
+ test('exhausts continuation retries without re-running the durable tool result', async () => {
const timers = manualWatchdogTimer();
const durable = durableTurnHarness('turn-1', 'read notes');
let providerCalls = 0;
@@ -11001,13 +11035,13 @@ describe('AiSdkBackend RunTrace', () => {
const events: SessionEvent[] = [];
const eventsPromise = collectEvents(backend.send(durable.input()), events, durable.record);
- await waitFor(() => providerCalls === 2);
- timers.fire();
- await waitFor(() => providerCalls === 3);
- timers.fire();
+ for (let request = 2; request <= 11; request += 1) {
+ await waitFor(() => providerCalls === request);
+ timers.fire();
+ }
await eventsPromise;
- assert.equal(providerCalls, 3);
+ assert.equal(providerCalls, 11);
assert.equal(toolCalls, 1);
assert.equal(events.filter((event) => event.type === 'tool_result').length, 1);
assert.equal(
@@ -11017,7 +11051,7 @@ describe('AiSdkBackend RunTrace', () => {
assert.equal(
events.filter((event) => event.type === 'provider_retry' && event.phase === 'scheduled')
.length,
- 1,
+ 9,
);
assert.equal(
events.find((event) => event.type === 'error')?.reason,
@@ -11088,7 +11122,7 @@ describe('AiSdkBackend RunTrace', () => {
assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'error');
});
- test('does not retry an idle watchdog timeout after partial answer text', async () => {
+ test('bounds repeated partial-answer timeouts by the shared request budget', async () => {
const timers = manualWatchdogTimer();
const traces: RunTraceEvent[] = [];
let calls = 0;
@@ -11123,10 +11157,11 @@ describe('AiSdkBackend RunTrace', () => {
if (event.type === 'text_delta' && event.text === 'partial answer') timers.fire();
}
- assert.equal(calls, 1);
+ assert.equal(calls, 10);
assert.equal(
- events.some((event) => event.type === 'provider_retry'),
- false,
+ events.filter((event) => event.type === 'provider_retry' && event.phase === 'scheduled')
+ .length,
+ 9,
);
assert.equal(events.find((event) => event.type === 'error')?.reason, 'timeout');
assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'error');
@@ -11282,6 +11317,7 @@ describe('AiSdkBackend RunTrace', () => {
});
test('does not retry after provider-executed tool input starts', async () => {
+ const durable = durableTurnHarness('turn-provider-tool-failure', 'hi');
const timers = manualWatchdogTimer();
let calls = 0;
const model = new MockLanguageModelV4({
@@ -11291,6 +11327,10 @@ describe('AiSdkBackend RunTrace', () => {
stream: hangingProviderStream(
[
{ type: 'stream-start', warnings: [] },
+ { type: 'reasoning-start', id: 'partial-thinking' },
+ { type: 'reasoning-delta', id: 'partial-thinking', delta: 'unfinished reasoning' },
+ { type: 'text-start', id: 'partial-text' },
+ { type: 'text-delta', id: 'partial-text', delta: 'unfinished answer' },
{
type: 'tool-input-start',
id: 'provider-tool-1',
@@ -11317,14 +11357,10 @@ describe('AiSdkBackend RunTrace', () => {
providerRetrySleep: async () => {},
});
- const events: SessionEvent[] = [];
- const eventsPromise = collectEvents(
- backend.send({ turnId: 'turn-1', text: 'hi', context: [] }),
- events,
- );
- await waitFor(() => timers.armCount() >= 4);
+ const eventsPromise = drainDurably(backend.send(durable.input()), durable);
+ await waitFor(() => timers.armCount() >= 8);
timers.fire();
- await eventsPromise;
+ const events = await eventsPromise;
assert.equal(calls, 1);
assert.equal(
@@ -11333,6 +11369,13 @@ describe('AiSdkBackend RunTrace', () => {
);
assert.equal(events.find((event) => event.type === 'error')?.reason, 'timeout');
assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'error');
+ const fragments = durable.ledger.filter(
+ (event) =>
+ event.role === 'model' &&
+ (event.content?.kind === 'text' || event.content?.kind === 'thinking'),
+ );
+ assert.equal(fragments.length, 2);
+ for (const fragment of fragments) assert.equal(fragment.modelVisibility, 'hidden');
});
test('does not retry an idle watchdog timeout after text continuation metadata', async () => {
@@ -13735,7 +13778,9 @@ describe('AiSdkBackend thinking persistence', () => {
}
assert.equal(events.find((event) => event.type === 'complete')?.stopReason, 'error');
- const parts = appended[0]?.thinking?.parts;
+ const parts = appended.flatMap(
+ (message) => message.thinking?.parts ?? (message.thinking ? [message.thinking] : []),
+ );
assert.deepEqual(
parts?.map((part) => [
part.text,
@@ -13746,6 +13791,21 @@ describe('AiSdkBackend thinking persistence', () => {
['unsafe item summary', undefined],
],
);
+ const backfilled = backfillRuntimeEventsFromStoredMessages({
+ run: { sessionId: 'session-1', invocationId: 'inv-1', runId: 'run-1', turnId: 'turn-1' },
+ messages: JSON.parse(JSON.stringify(appended)),
+ });
+ assert.deepEqual(
+ backfilled.events.flatMap((event) =>
+ event.content?.kind === 'thinking'
+ ? [[event.content.text, event.modelVisibility ?? 'visible']]
+ : [],
+ ),
+ [
+ ['valid summary', 'visible'],
+ ['unsafe item summary', 'hidden'],
+ ],
+ );
const ctx = {
sessionId: 'session-1',
@@ -13873,10 +13933,14 @@ describe('AiSdkBackend thinking persistence', () => {
}
assert.deepEqual(
- appended[0]?.thinking?.parts?.map((part) => [
- part.text,
- (part.providerOptions?.makaResponses as { itemId?: unknown } | undefined)?.itemId,
- ]),
+ appended
+ .flatMap(
+ (message) => message.thinking?.parts ?? (message.thinking ? [message.thinking] : []),
+ )
+ .map((part) => [
+ part.text,
+ (part.providerOptions?.makaResponses as { itemId?: unknown } | undefined)?.itemId,
+ ]),
[
['valid summary', 'reasoning-item-a'],
['late duplicate', undefined],
diff --git a/packages/runtime/src/__tests__/model-adapter.test.ts b/packages/runtime/src/__tests__/model-adapter.test.ts
index a56a940153..710af62b2a 100644
--- a/packages/runtime/src/__tests__/model-adapter.test.ts
+++ b/packages/runtime/src/__tests__/model-adapter.test.ts
@@ -481,7 +481,7 @@ describe('ModelAdapter stream and error normalization', () => {
]);
});
- test('surfaces provider-executed tool input as replay-unsafe activity', () => {
+ test('preserves local input sampling separately from provider tool activity', () => {
const adapter = newAdapter();
type Chunk = Parameters<typeof adapter.translateChunk>[0];
@@ -492,7 +492,7 @@ describe('ModelAdapter stream and error normalization', () => {
toolName: 'WebSearch',
providerExecuted: true,
} as Chunk),
- [{ kind: 'provider-tool-input' }],
+ [{ kind: 'tool-input', providerExecuted: true }],
);
assert.deepEqual(
adapter.translateChunk({
@@ -501,7 +501,7 @@ describe('ModelAdapter stream and error normalization', () => {
toolName: 'Read',
providerExecuted: false,
} as Chunk),
- [],
+ [{ kind: 'tool-input', providerExecuted: false }],
);
});
diff --git a/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts b/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts
index 8d8119456b..800b43d395 100644
--- a/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts
+++ b/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts
@@ -124,6 +124,7 @@ type CallKind =
| 'terminated'
| 'terminatedMidBody'
| 'partialThenTerminated'
+ | 'partialToolInputThenOverflow'
| 'partialThenOverflowPart';
const RETRY_STEP_TEXT_SENTINEL = 'RETRY_STEP_TEXT_SENTINEL reasoning before the big read';
@@ -349,6 +350,18 @@ function buildReactiveFixture(options: ReactiveFixtureOptions): ReactiveFixture
chunkDelayInMs: null,
});
}
+ if (kind === 'partialToolInputThenOverflow') {
+ return simulateReadableStream({
+ chunks: [
+ { type: 'stream-start', warnings: [] },
+ { type: 'tool-input-start', id: 'unfinished', toolName: 'Read' },
+ { type: 'tool-input-delta', id: 'unfinished', delta: '{"path":"' },
+ { type: 'error', error: { message: 'Bad Request', code: 'context_length_exceeded' } },
+ ],
+ initialDelayInMs: null,
+ chunkDelayInMs: null,
+ });
+ }
if (
kind === 'overflowPart' ||
kind === 'overflowPartResponses' ||
@@ -1136,28 +1149,50 @@ describe('reactive overflow recovery in the streaming backend', () => {
assert.deepEqual(fixture.toolExecutions, ['one.md']);
});
- test('does not retry a terminated request after visible output was emitted', async () => {
+ test('preserves interrupted text while retrying from the completed tool step', async () => {
const fixture = buildReactiveFixture({ script: ['tool', 'partialThenTerminated', 'done'] });
await runTurn(fixture);
- assert.equal(fixture.model.doStreamCalls.length, 2);
- assert.equal(complete(fixture)?.stopReason, 'error');
+ assert.equal(fixture.model.doStreamCalls.length, 3);
+ assert.equal(complete(fixture)?.stopReason, 'end_turn');
+ assert.deepEqual(fixture.toolExecutions, ['one.md']);
+ assert.equal(JSON.stringify(fixture.model.doStreamCalls[2]?.prompt).includes('partial'), false);
assert.equal(
- fixture.events.some((event) => event.type === 'text_delta' && event.text === 'partial'),
+ fixture.events.some(
+ (event) => event.type === 'text_complete' && event.text === 'partial' && event.interrupted,
+ ),
true,
);
});
- test('surfaces the tenth transport failure without spending another sample', async () => {
+ test('counts overflow recovery within the ten-request budget', async () => {
+ const failures = Array.from({ length: 10 }, () => 'terminated' as const);
+ const scripts: CallKind[][] = [
+ ['overflow', ...failures],
+ [...failures.slice(1), 'overflow', 'done'],
+ ];
+ for (const script of scripts) {
+ const fixture = buildReactiveFixture({
+ script: ['tool', ...script],
+ });
+ await runTurn(fixture);
+
+ assert.equal(fixture.model.doStreamCalls.length, 11);
+ assert.equal(complete(fixture)?.stopReason, 'error');
+ assert.deepEqual(fixture.toolExecutions, ['one.md']);
+ assert.equal(fixture.recorded.length, script[0] === 'overflow' ? 1 : 0);
+ }
+ });
+
+ test('recovers after partial tool arguments without treating missing usage as complete', async () => {
const fixture = buildReactiveFixture({
- script: ['tool', ...Array.from({ length: 10 }, () => 'terminated' as const)],
+ script: ['tool', 'partialToolInputThenOverflow', 'done'],
});
await runTurn(fixture);
-
- assert.equal(fixture.model.doStreamCalls.length, 11);
- assert.equal(complete(fixture)?.stopReason, 'error');
+ assert.equal(complete(fixture)?.stopReason, 'end_turn');
+ assert.equal(fixture.model.doStreamCalls.length, 3);
assert.deepEqual(fixture.toolExecutions, ['one.md']);
- assert.equal(fixture.recorded.length, 0);
+ assert.equal(fixture.llmCalls.length, 0);
});
test('compacts once and retries after a mid-stream context-length overflow', async () => {
@@ -1590,18 +1625,18 @@ describe('reactive overflow recovery in the streaming backend', () => {
assert.equal(fixture.recorded[0]!.phase, 'mid_turn');
});
- test('does not recover an overflow after the failed stream emitted a tool call', async () => {
+ test('discards an undispatched tool call before overflow compaction recovery', async () => {
const fixture = buildReactiveFixture({
script: ['tool', 'toolThenOverflowPart', 'done'],
bigPriors: true,
});
await runTurn(fixture);
- assert.equal(fixture.model.doStreamCalls.length, 2);
- assert.equal(complete(fixture)?.stopReason, 'error');
+ assert.equal(fixture.model.doStreamCalls.length, 3);
+ assert.equal(complete(fixture)?.stopReason, 'end_turn');
assert.deepEqual(fixture.toolExecutions, ['one.md']);
- assert.equal(fixture.recorded.length, 0);
- assert.equal(fixture.summarizerCalls(), 0);
+ assert.equal(fixture.recorded.length, 1);
+ assert.equal(fixture.summarizerCalls(), 1);
});
test('does not recover an overflow after the failed stream emitted visible text', async () => {
diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts
index 471340daac..a515fe02bd 100644
--- a/packages/runtime/src/ai-sdk-compaction.ts
+++ b/packages/runtime/src/ai-sdk-compaction.ts
@@ -1208,7 +1208,6 @@ export class AiSdkCompaction {
*/
public async recoverFromOverflowError(input: {
error: unknown;
- retryAlreadyUsed: boolean;
midTurnState: MidTurnCapacityCompactState | undefined;
turnId: string;
stepNumber: number;
@@ -1222,7 +1221,9 @@ export class AiSdkCompaction {
abortSignal?: AbortSignal;
}): Promise<{ messages: ModelMessage[] } | undefined> {
const state = input.midTurnState;
- if (input.retryAlreadyUsed || !state) return undefined;
+ if (!state || state.compactionAttemptedThisSend || state.omittedImageToolResults.size > 0) {
+ return undefined;
+ }
if (this.modelAdapter.classifyError(input.error) !== 'context_overflow') return undefined;
const eligibleImages = collectHistoricalImageToolResults(state.priorContentEvents);
diff --git a/packages/runtime/src/ai-sdk-message-projection.ts b/packages/runtime/src/ai-sdk-message-projection.ts
index 4cbe71c20f..46e6efbe6a 100644
--- a/packages/runtime/src/ai-sdk-message-projection.ts
+++ b/packages/runtime/src/ai-sdk-message-projection.ts
@@ -18,6 +18,7 @@
*/
import type { AttachmentRef, DirectoryReference, QuoteRef, StorageRef } from '@maka/core/events';
+import type { AssistantThinkingPart } from '@maka/core/session';
import {
MAX_PROVIDER_IMAGE_REQUEST_BYTES,
PROVIDER_IMAGE_BUDGET_EXCEEDED_MESSAGE,
@@ -68,6 +69,42 @@ export interface AiSdkMessageProjectionInput {
maxProviderImageRequestBytes?: number;
}
+function isRedactedThinking(providerOptions: AssistantThinkingPart['providerOptions']): boolean {
+ const anthropic = providerOptions?.anthropic;
+ return (
+ !!anthropic &&
+ typeof anthropic === 'object' &&
+ !Array.isArray(anthropic) &&
+ typeof (anthropic as { redactedData?: unknown }).redactedData === 'string'
+ );
+}
+
+function encryptedResponsesReasoning(
+ providerOptions: AssistantThinkingPart['providerOptions'],
+): { itemId: string; reasoningEncryptedContent: string } | undefined {
+ const openai = providerOptions?.openai;
+ if (!openai || typeof openai !== 'object' || Array.isArray(openai)) return undefined;
+ const { itemId, reasoningEncryptedContent } = openai as {
+ itemId?: unknown;
+ reasoningEncryptedContent?: unknown;
+ };
+ return typeof itemId === 'string' &&
+ itemId.length > 0 &&
+ typeof reasoningEncryptedContent === 'string' &&
+ reasoningEncryptedContent.length > 0
+ ? { itemId, reasoningEncryptedContent }
+ : undefined;
+}
+
+export function hasFinalizedReasoning(part: AssistantThinkingPart): boolean {
+ return (
+ !!part.signature ||
+ isRedactedThinking(part.providerOptions) ||
+ decodePlaintextResponsesReasoningState(part.providerOptions).kind === 'valid' ||
+ encryptedResponsesReasoning(part.providerOptions) !== undefined
+ );
+}
+
function isImageToolResult(
value: unknown,
): value is { kind: 'image'; mimeType: string; ref: StorageRef } {
@@ -231,13 +268,7 @@ export class AiSdkMessageProjection {
}
: undefined;
}
- const anthropic = item.providerOptions?.anthropic;
- if (
- anthropic &&
- typeof anthropic === 'object' &&
- !Array.isArray(anthropic) &&
- typeof (anthropic as { redactedData?: unknown }).redactedData === 'string'
- ) {
+ if (isRedactedThinking(item.providerOptions)) {
return replaySupport.signedThinking
? {
part: {
@@ -285,31 +316,17 @@ export class AiSdkMessageProjection {
return { part: { type: 'reasoning' as const, text: item.text } };
}
if (replaySupport.responsesReasoning === 'encrypted-content') {
- const openai = item.providerOptions?.openai;
- if (openai && typeof openai === 'object' && !Array.isArray(openai)) {
- const { itemId, reasoningEncryptedContent } = openai as {
- itemId?: unknown;
- reasoningEncryptedContent?: unknown;
- };
- if (
- typeof itemId === 'string' &&
- itemId.length > 0 &&
- typeof reasoningEncryptedContent === 'string' &&
- reasoningEncryptedContent.length > 0
- ) {
- return {
- part: {
- type: 'reasoning' as const,
- text: item.text,
- providerOptions: {
- openai: {
- itemId,
- reasoningEncryptedContent,
- },
- },
+ const encrypted = encryptedResponsesReasoning(item.providerOptions);
+ if (encrypted) {
+ return {
+ part: {
+ type: 'reasoning' as const,
+ text: item.text,
+ providerOptions: {
+ openai: encrypted,
},
- };
- }
+ },
+ };
}
}
if (!replaySupport.unsignedThinking) return undefined;
diff --git a/packages/runtime/src/ai-sdk-turn.ts b/packages/runtime/src/ai-sdk-turn.ts
index 7e11505dad..382188da4a 100644
--- a/packages/runtime/src/ai-sdk-turn.ts
+++ b/packages/runtime/src/ai-sdk-turn.ts
@@ -41,7 +41,6 @@ import type {
} from '@maka/core/events';
import type {
AssistantMessage,
- AssistantStepContentKind,
AssistantThinkingPart,
RuntimeSystemNoteKind,
SessionHeader,
@@ -148,7 +147,7 @@ import {
} from './request-shape.js';
import { toolAvailabilityHash } from './tool-availability.js';
import { ProviderRequestTelemetry } from './provider-request-telemetry.js';
-import { AiSdkMessageProjection } from './ai-sdk-message-projection.js';
+import { AiSdkMessageProjection, hasFinalizedReasoning } from './ai-sdk-message-projection.js';
import { ToolAvailabilityRuntime, type ToolAvailabilityPlan } from './tool-availability.js';
import { renderSwarmModePrompt } from './swarm-mode.js';
import { renderGraphModePrompt } from './graph-mode.js';
@@ -573,13 +572,6 @@ function joinPromptFragments(fragments: readonly (string | undefined)[]): string
const MAX_WAITING_CODE_MODE_CELLS = 1;
const MAX_PROVIDER_ATTEMPTS_PER_STEP = 10;
-const MAX_IDLE_WATCHDOG_RETRIES_PER_STEP = 1;
-const MAX_INCOMPLETE_STREAM_RETRIES_PER_STEP = 1;
-// A mid-stream cut after partial thinking seals one transcript fragment per
-// retry. A gateway that systematically kills long thinking streams (the
-// 2026-08-28 incident shape) would otherwise spend the full attempt budget
-// accumulating fragments before failing anyway, so fail fast after one.
-const MAX_SEALED_THINKING_RETRIES_PER_STEP = 1;
const PROVIDER_RETRY_BASE_DELAY_MS = 1_000;
const PROVIDER_RETRY_MAX_DELAY_MS = 32_000;
const PROVIDER_RETRY_JITTER_FACTOR = 0.25;
@@ -890,10 +882,6 @@ export class AiSdkTurn {
let stepTextPartStartOffset = 0;
let stepThinkingParts: AssistantThinkingPart[] = [];
let stepThinkingPartsById = new Map<string, AssistantThinkingPart>();
- let stepContentOrder: AssistantStepContentKind[] = [];
- const recordStepContent = (kind: AssistantStepContentKind): void => {
- if (!stepContentOrder.includes(kind)) stepContentOrder.push(kind);
- };
// Flush the current step's AssistantMessage (text + thinking) and the paired
// terminal thinking/text events, then clear the per-step accumulators.
// Persist when the step produced text OR reasoning — a thinking-only step
@@ -909,17 +897,38 @@ export class AiSdkTurn {
stepTextPartStartOffset = 0;
stepThinkingParts = [];
stepThinkingPartsById = new Map();
- stepContentOrder = [];
};
- const flushStep = async (): Promise<void> => {
+ const flushStep = async (interrupted = false): Promise<void> => {
const hasThinking = stepThinkingParts.length > 0;
if (stepText.length === 0 && !hasThinking) {
resetStep();
return;
}
- const stepId = currentStepMessageId;
- if (hasThinking) {
- for (const part of stepThinkingParts) {
+ // A provider may finalize one reasoning item before the next item fails.
+ // Keep completed and interrupted parts in separate transcript rows so
+ // missing-ledger recovery preserves the same model visibility.
+ const fragments: { thinking: AssistantThinkingPart[]; text: string; interrupted: boolean }[] =
+ [];
+ for (const part of stepThinkingParts) {
+ const partInterrupted = interrupted && !hasFinalizedReasoning(part);
+ let fragment = fragments.at(-1);
+ if (!fragment || fragment.interrupted !== partInterrupted) {
+ fragment = { thinking: [], text: '', interrupted: partInterrupted };
+ fragments.push(fragment);
+ }
+ fragment.thinking.push(part);
+ }
+ if (stepText.length > 0) {
+ let fragment = fragments.at(-1);
+ if (!fragment || fragment.interrupted !== interrupted) {
+ fragment = { thinking: [], text: '', interrupted };
+ fragments.push(fragment);
+ }
+ fragment.text = stepText;
+ }
+ for (const [index, fragment] of fragments.entries()) {
+ const stepId = index === 0 ? currentStepMessageId : this.deps.newId();
+ for (const part of fragment.thinking) {
queue.push({
type: 'thinking_complete',
id: this.deps.newId(),
@@ -927,6 +936,7 @@ export class AiSdkTurn {
ts: this.deps.now(),
messageId: stepId,
text: part.text,
+ ...(fragment.interrupted ? { interrupted: true } : {}),
...(part.signature !== undefined ? { signature: part.signature } : {}),
// No sanitiser here, unlike the tool call below: these options are
// not the provider's object. `translateChunk` rebuilds reasoning
@@ -939,18 +949,19 @@ export class AiSdkTurn {
: {}),
} satisfies ThinkingCompleteEvent);
}
+ queue.push({
+ type: 'text_complete',
+ id: this.deps.newId(),
+ turnId,
+ ts: this.deps.now(),
+ messageId: stepId,
+ text: fragment.text,
+ ...(fragment.interrupted ? { interrupted: true } : {}),
+ ...(index === fragments.length - 1 && stepTextProviderOptions !== undefined
+ ? { providerOptions: stepTextProviderOptions }
+ : {}),
+ } satisfies TextCompleteEvent);
}
- queue.push({
- type: 'text_complete',
- id: this.deps.newId(),
- turnId,
- ts: this.deps.now(),
- messageId: stepId,
- text: stepText,
- ...(stepTextProviderOptions !== undefined
- ? { providerOptions: stepTextProviderOptions }
- : {}),
- } satisfies TextCompleteEvent);
this.finalAssistantText = stepText.length > 0 ? stepText : undefined;
resetStep();
};
@@ -1437,11 +1448,6 @@ export class AiSdkTurn {
const completedProviderSteps: RequestProjectionContext['completedSteps'][number][] = [];
let requestMessages: ModelMessage[] = messages;
- // The compaction module runs at most once per send. This tracks the
- // reactive entry; the proactive one sets the same flag on the mid-turn
- // state, and each consults the other, so a send that already folded
- // reports the oversized message instead of folding again (#4559).
- let overflowRetryUsed = false;
let result: ModelStreamResult;
let providerOutcome: ModelStepOutcome;
let finishReason: ModelFinishReason = 'stop';
@@ -1533,43 +1539,28 @@ export class AiSdkTurn {
: undefined;
providerRequestTracker?.setStep(runtimeSteps, requestCompositionId);
let attemptMessages = projectedMessages;
- let providerAttempt = 1;
- let idleWatchdogRetryCount = 0;
- let incompleteStreamRetryCount = 0;
- let sealedThinkingRetryCount = 0;
+ let providerAttempt = 0;
const returnedToolCalls: ToolCallPart[] = [];
- let providerToolActivityCount = 0;
const providerToolInputs = new Map<string, unknown>();
let providerStepUsage: NormalizedUsage | undefined;
for (;;) {
+ providerAttempt += 1;
+ // Local calls are only admitted after a successful provider outcome.
+ // A new physical request must not inherit the failed one's intents.
+ returnedToolCalls.length = 0;
providerRequestAbortController = new AbortController();
watchdogTimeoutState.current = null;
startWatchdog();
// Monotonic facts for this physical request. The step accumulators
// are cleared after flushStep(), so they cannot decide whether a
// later stream failure is safe to retry.
- let attemptSawText = false;
- let attemptSawThinking = false;
+ let attemptSawVisibleContent = false;
let attemptSawToolActivity = false;
- let attemptSawContinuationMetadata = false;
- let attemptReachedStepBoundary = false;
+ let attemptSawToolInput = false;
+ let attemptSawReplayBarrier = false;
const attemptHasNoObservableOutput = () =>
- !attemptSawText &&
- !attemptSawThinking &&
- !attemptSawToolActivity &&
- !attemptSawContinuationMetadata &&
- !attemptReachedStepBoundary;
- // Thinking is the only output that can be sealed into its own
- // message before a retry: flushStep() closes the fragment under
- // the current message id and the retry streams into a fresh one,
- // so the user never sees spliced or duplicated content. Text,
- // tool activity, continuation metadata, and step boundaries stay
- // non-recoverable for the reasons each of them is tracked.
- const attemptCanRecoverWithSealedThinking = () =>
- !attemptSawText &&
- !attemptSawToolActivity &&
- !attemptSawContinuationMetadata &&
- !attemptReachedStepBoundary;
+ !attemptSawVisibleContent && !attemptSawToolActivity && !attemptSawReplayBarrier;
+ const attemptCanReplay = () => !attemptSawToolActivity && !attemptSawReplayBarrier;
this.memorySourceMessages = [...attemptMessages];
this.memorySourceEventMessagePositions =
this.deps.messageProjection.memoryEventMessagePositions(attemptMessages);
@@ -1634,7 +1625,7 @@ export class AiSdkTurn {
(event.kind === 'finish' || event.kind === 'step-finish') &&
isIncompleteProviderFinishReason(event.finishReason);
if ((event.kind === 'finish' || event.kind === 'step-finish') && !incompleteFinish) {
- attemptReachedStepBoundary = true;
+ attemptSawReplayBarrier = true;
}
if (event.kind === 'step-finish') {
// AI SDK can synthesize `finish-step(other)` when the provider
@@ -1814,14 +1805,14 @@ export class AiSdkTurn {
}
if (event.kind === 'text-start') {
if (stepText.length > 0 && event.providerItemBoundary === true) {
+ attemptSawReplayBarrier = true;
await flushStep();
currentStepMessageId = this.deps.newId();
}
stepTextPartStartOffset = stepText.length;
} else if (event.kind === 'text') {
- if (event.text.length > 0) recordStepContent('text');
stepText += event.text;
- if (event.text.length > 0) attemptSawText = true;
+ if (event.text.length > 0) attemptSawVisibleContent = true;
queue.push({
type: 'text_delta',
id: this.deps.newId(),
@@ -1832,7 +1823,7 @@ export class AiSdkTurn {
} satisfies TextDeltaEvent);
} else if (event.kind === 'text-end') {
if (event.providerOptions !== undefined) {
- attemptSawContinuationMetadata = true;
+ attemptSawReplayBarrier = true;
stepTextProviderOptions = mergeTextProviderOptions(
stepTextProviderOptions,
stripUndefinedDeep(event.providerOptions) as NonNullable<
@@ -1847,7 +1838,7 @@ export class AiSdkTurn {
}
} else if (event.kind === 'thinking-start') {
if (event.providerOptions !== undefined) {
- attemptSawContinuationMetadata = true;
+ attemptSawReplayBarrier = true;
}
const part: AssistantThinkingPart = {
text: '',
@@ -1860,11 +1851,10 @@ export class AiSdkTurn {
stepThinkingPartsById.set(event.reasoningPartId, part);
}
} else if (event.kind === 'thinking') {
- if (event.text.length > 0) recordStepContent('thinking');
- if (event.text.length > 0) attemptSawThinking = true;
+ if (event.text.length > 0) attemptSawVisibleContent = true;
if (event.providerOptions !== undefined) {
if (event.providerOptionsOrigin !== 'maka_transport') {
- attemptSawContinuationMetadata = true;
+ attemptSawReplayBarrier = true;
}
}
const partId =
@@ -1927,7 +1917,7 @@ export class AiSdkTurn {
text: event.text,
} satisfies ThinkingDeltaEvent);
} else if (event.kind === 'thinking-signature') {
- attemptSawContinuationMetadata = true;
+ attemptSawReplayBarrier = true;
let part = event.reasoningPartId
? stepThinkingPartsById.get(event.reasoningPartId)
: stepThinkingParts.at(-1);
@@ -1939,16 +1929,15 @@ export class AiSdkTurn {
}
}
part.signature = event.signature;
- } else if (event.kind === 'provider-tool-input') {
+ } else if (event.kind === 'tool-input') {
+ attemptSawToolInput = true;
// The provider has started its own tool. Even without a
// final tool-call/result event, retrying can repeat external
// work that the Runtime cannot observe or reconcile.
- attemptSawToolActivity = true;
+ if (event.providerExecuted) attemptSawToolActivity = true;
} else if (event.kind === 'tool-call') {
- attemptSawToolActivity = true;
- recordStepContent('tools');
if (event.toolCall.providerExecuted) {
- providerToolActivityCount += 1;
+ attemptSawToolActivity = true;
providerToolInputs.set(event.toolCall.toolCallId, event.toolCall.input);
queue.push({
type: 'tool_start',
@@ -1973,7 +1962,6 @@ export class AiSdkTurn {
}
} else if (event.kind === 'provider-tool-result') {
attemptSawToolActivity = true;
- providerToolActivityCount += 1;
const providerOutput = stripUndefinedDeep(event.output);
queue.push({
type: 'tool_result',
@@ -2016,12 +2004,6 @@ export class AiSdkTurn {
const settledWatchdogTimeout = consumeWatchdogTimeout();
providerOutcome = await result.outcome;
const incompleteStreamTerminal = providerOutcome.kind === 'truncated';
- const incompleteStreamHasNoObservableOutput =
- incompleteStreamTerminal &&
- !attemptSawText &&
- !attemptSawThinking &&
- !attemptSawToolActivity &&
- !attemptSawContinuationMetadata;
const attemptFailure =
settledWatchdogTimeout?.error ??
(providerOutcome.kind === 'completed' ? undefined : providerOutcome.failure);
@@ -2031,6 +2013,18 @@ export class AiSdkTurn {
settledWatchdogTimeout || providerOutcome.kind === 'completed'
? this.deps.modelAdapter.normalizeFailure(attemptFailure)
: providerOutcome.failure;
+ // An output-free context rejection did not sample a response.
+ // Preserve metering for completed steps across compaction recovery.
+ if (
+ !providerOutcome.usage &&
+ !(
+ failure.kind === 'context_overflow' &&
+ attemptHasNoObservableOutput() &&
+ !attemptSawToolInput &&
+ returnedToolCalls.length === 0
+ )
+ )
+ sawUnusableStepUsage = true;
if (this.loopStopRequested) {
terminalProviderError = settledWatchdogTimeout?.error ?? failure;
terminalProviderErrorReason =
@@ -2044,11 +2038,11 @@ export class AiSdkTurn {
// nothing left to grant it, so the error is terminal.
const stepBudgetRemains = maxSteps === undefined || runtimeSteps < maxSteps;
const recovered =
- stepBudgetRemains && attemptHasNoObservableOutput()
+ stepBudgetRemains &&
+ providerAttempt < MAX_PROVIDER_ATTEMPTS_PER_STEP &&
+ attemptHasNoObservableOutput()
? await this.deps.compaction.recoverFromOverflowError({
error: attemptFailure,
- retryAlreadyUsed:
- overflowRetryUsed || (midTurnState?.compactionAttemptedThisSend ?? false),
midTurnState,
turnId,
stepNumber: runtimeSteps,
@@ -2069,7 +2063,6 @@ export class AiSdkTurn {
})
: undefined;
if (recovered) {
- overflowRetryUsed = true;
attemptMessages = recovered.messages;
continue;
}
@@ -2112,37 +2105,10 @@ export class AiSdkTurn {
contextOverflowAfterCompactionNoteWritten = true;
await this.recordSystemNote('context_overflow_after_compaction', turnId);
}
- const idleWatchdogRecovery =
- settledWatchdogTimeout?.phase === 'idle' &&
- idleWatchdogRetryCount < MAX_IDLE_WATCHDOG_RETRIES_PER_STEP &&
- attemptCanRecoverWithSealedThinking();
- const incompleteStreamRecovery =
- incompleteStreamTerminal &&
- incompleteStreamRetryCount < MAX_INCOMPLETE_STREAM_RETRIES_PER_STEP &&
- incompleteStreamHasNoObservableOutput;
- // Same seal-and-retry contract as the watchdog path, entered when
- // the failure arrives as a retryable provider/network error
- // instead of a local idle timeout. `!idleWatchdogRecovery` keeps
- // every watchdog-shaped outcome on its existing path, and
- // `!attemptHasNoObservableOutput()` keeps no-output retries on
- // the plain budget so this one is spent only on sealed fragments.
- const sealedThinkingRecovery =
- !idleWatchdogRecovery &&
- failure.retryable &&
- sealedThinkingRetryCount < MAX_SEALED_THINKING_RETRIES_PER_STEP &&
- attemptCanRecoverWithSealedThinking() &&
- !attemptHasNoObservableOutput();
// The stopping gate also supplies the durable reason. An absent
// decision means this attempt is allowed to retry.
let retry: import('@maka/core/model-failure').ModelRetryDecision | undefined;
- if (
- !(
- attemptHasNoObservableOutput() ||
- idleWatchdogRecovery ||
- incompleteStreamRecovery ||
- sealedThinkingRecovery
- )
- ) {
+ if (!attemptCanReplay()) {
retry = {
decision: 'declined',
because: attemptSawToolActivity ? 'side_effects' : 'observable_output',
@@ -2153,33 +2119,22 @@ export class AiSdkTurn {
retry = { decision: 'exhausted', attempts: providerAttempt };
} else if (failure.kind === 'context_overflow') {
retry = { decision: 'declined', because: 'policy' };
- } else if (!(failure.retryable || idleWatchdogRecovery || incompleteStreamRecovery)) {
- retry =
- incompleteStreamTerminal &&
- incompleteStreamRetryCount >= MAX_INCOMPLETE_STREAM_RETRIES_PER_STEP
- ? { decision: 'exhausted', attempts: providerAttempt }
- : { decision: 'declined', because: 'policy' };
+ } else if (
+ !(failure.retryable || settledWatchdogTimeout || incompleteStreamTerminal)
+ ) {
+ retry = { decision: 'declined', because: 'policy' };
}
if (!retry) {
- if (idleWatchdogRecovery) idleWatchdogRetryCount += 1;
- if (sealedThinkingRecovery) sealedThinkingRetryCount += 1;
- if (incompleteStreamRecovery) incompleteStreamRetryCount += 1;
- if (
- (idleWatchdogRecovery || sealedThinkingRecovery) &&
- stepThinkingParts.length > 0
- ) {
- await flushStep();
- currentStepMessageId = this.deps.newId();
- }
+ // Preserve the failed fragment for the transcript, but exclude
+ // it from model history when a later tool step reloads the ledger.
+ await flushStep(true);
+ currentStepMessageId = this.deps.newId();
// The failed request did not return authoritative usage. Keep
// effectiveness recoverable, but fail final metering closed.
sawUnusableStepUsage = true;
const delayMs = providerRetryDelayMs(providerAttempt, failure.retryAfterMs);
const nextAttempt = providerAttempt + 1;
- const maxAttempts =
- idleWatchdogRecovery || incompleteStreamRecovery || sealedThinkingRecovery
- ? nextAttempt
- : MAX_PROVIDER_ATTEMPTS_PER_STEP;
+ const maxAttempts = MAX_PROVIDER_ATTEMPTS_PER_STEP;
const reason = providerRetryReason(failure.kind);
queue.push({
type: 'provider_retry',
@@ -2194,14 +2149,13 @@ export class AiSdkTurn {
reason,
} satisfies ProviderRetryEvent);
await this.deps.providerRetrySleep(delayMs, turnAbortController.signal);
- providerAttempt = nextAttempt;
queue.push({
type: 'provider_retry',
id: this.deps.newId(),
turnId,
ts: this.deps.now(),
phase: 'started',
- attempt: providerAttempt,
+ attempt: nextAttempt,
maxAttempts,
reason,
} satisfies ProviderRetryEvent);
@@ -2573,7 +2527,7 @@ export class AiSdkTurn {
// `finish-step`; this keeps their and this step's streamed-out output on
// BOTH exits — user stop and provider error / watchdog timeout — so the
// transcript keeps what the user actually saw.
- await flushStep().catch(() => {});
+ await flushStep(!this.aborted).catch(() => {});
if (this.aborted) {
queue.push({
type: 'abort',
diff --git a/packages/runtime/src/model-adapter.ts b/packages/runtime/src/model-adapter.ts
index f211730846..29aec059e4 100644
--- a/packages/runtime/src/model-adapter.ts
+++ b/packages/runtime/src/model-adapter.ts
@@ -1082,7 +1082,7 @@ function translateChunk(
case 'tool-input-start':
case 'tool-input-delta':
case 'tool-input-end':
- return chunk.providerExecuted === true ? [{ kind: 'provider-tool-input' }] : [];
+ return [{ kind: 'tool-input', providerExecuted: chunk.providerExecuted === true }];
// Step boundaries (`start-step` / `finish-step`) and the terminal `finish`
// carry no text/thinking to stream. The backend owns step accounting: it
// counts and flushes one AssistantMessage per step and rotates the
diff --git a/packages/runtime/src/model-protocol.ts b/packages/runtime/src/model-protocol.ts
index 06c36c60eb..e192cc676f 100644
--- a/packages/runtime/src/model-protocol.ts
+++ b/packages/runtime/src/model-protocol.ts
@@ -406,8 +406,8 @@ export type ModelStreamEvent =
providerOptionsOrigin?: 'maka_transport';
}
| { kind: 'thinking-signature'; signature: string; reasoningPartId?: string }
- /** Provider-side tool execution has begun, but no replayable call exists yet. */
- | { kind: 'provider-tool-input' }
+ /** Tool input was sampled; only providerExecuted also implies external activity. */
+ | { kind: 'tool-input'; providerExecuted: boolean }
| { kind: 'tool-call'; toolCall: ToolCallPart }
| {
kind: 'provider-tool-result';
diff --git a/packages/runtime/src/runtime-event-backfill.ts b/packages/runtime/src/runtime-event-backfill.ts
index df7a49d894..be32b644d6 100644
--- a/packages/runtime/src/runtime-event-backfill.ts
+++ b/packages/runtime/src/runtime-event-backfill.ts
@@ -163,7 +163,12 @@ export function backfillRuntimeEventsFromStoredMessages(
id: newId(),
role: 'model',
author: 'agent',
- content: { kind: 'text', text: message.text },
+ ...(message.interrupted ? { modelVisibility: 'hidden' as const } : {}),
+ content: {
+ kind: 'text',
+ text: message.text,
+ ...(message.interrupted ? { interrupted: true } : {}),
+ },
actions: { stateDelta: recoveryState(now, message) },
refs: { storedMessageId: message.id },
});
@@ -173,6 +178,7 @@ export function backfillRuntimeEventsFromStoredMessages(
for (const part of parts) {
events.push({
...base,
+ ...(message.interrupted ? { modelVisibility: 'hidden' as const } : {}),
id: newId(),
role: 'model',
author: 'agent',
diff --git a/packages/runtime/src/runtime-event-read-model.ts b/packages/runtime/src/runtime-event-read-model.ts
index 4bc9081196..9201dbbb48 100644
--- a/packages/runtime/src/runtime-event-read-model.ts
+++ b/packages/runtime/src/runtime-event-read-model.ts
@@ -955,6 +955,7 @@ function projectText(
turnId: event.turnId,
ts: event.ts,
text: event.content.text,
+ ...(event.content.interrupted ? { interrupted: true } : {}),
...(event.content.providerOptions !== undefined
? { providerOptions: structuredClone(event.content.providerOptions) }
: {}),
diff --git a/packages/runtime/src/session-event-runtime-mapper.ts b/packages/runtime/src/session-event-runtime-mapper.ts
index 6d5253c1b3..4a00ece1c1 100644
--- a/packages/runtime/src/session-event-runtime-mapper.ts
+++ b/packages/runtime/src/session-event-runtime-mapper.ts
@@ -210,11 +210,13 @@ function mapBackendSessionEvent(
case 'text_complete':
return {
...base,
+ ...(event.interrupted ? { modelVisibility: 'hidden' as const } : {}),
role: 'model',
author: 'agent',
content: {
kind: 'text',
text: event.text,
+ ...(event.interrupted ? { interrupted: true } : {}),
...(event.providerOptions !== undefined
? { providerOptions: structuredClone(event.providerOptions) }
: {}),
@@ -235,6 +237,7 @@ function mapBackendSessionEvent(
case 'thinking_complete':
return {
...base,
+ ...(event.interrupted ? { modelVisibility: 'hidden' as const } : {}),
role: 'model',
author: 'agent',
content: {
diff --git a/packages/ui/src/__tests__/chat-turn-steering-order.test.ts b/packages/ui/src/__tests__/chat-turn-steering-order.test.ts
index 87a8cf0ca1..e97578743f 100644
--- a/packages/ui/src/__tests__/chat-turn-steering-order.test.ts
+++ b/packages/ui/src/__tests__/chat-turn-steering-order.test.ts
@@ -24,7 +24,7 @@ import { renderToStaticMarkup } from 'react-dom/server';
import { parseHTML } from 'linkedom';
import { TurnView } from '../chat-turn.js';
import { LocaleProvider } from '../locale-context.js';
-import type { TurnViewModel } from '../materialize.js';
+import { materializeTurns, type TurnViewModel } from '../materialize.js';
import { createTranscriptProjection } from '../transcript-projection.js';
import { ChatView } from '../chat-view.js';
import { Composer } from '../composer.js';
@@ -33,6 +33,22 @@ import { armLiveTurn } from '../live-turn-projection.js';
import { applyLiveTurnEvent } from './live-turn-zh.js';
import type { SessionSummary, StoredMessage } from '@maka/core/session';
+test('renders a thinking-only interruption as a divider without an empty answer bubble', () => {
+ const messages: StoredMessage[] = [
+ { type: 'user', id: 'user', turnId: 'turn', ts: 1, text: 'request' },
+ { type: 'assistant', id: 'partial', turnId: 'turn', ts: 2, modelId: 'mock', text: '', interrupted: true, thinking: { text: 'partial thought' } },
+ ];
+ const [turn] = materializeTurns(messages, 'en');
+ assert.ok(turn);
+ const markup = renderToStaticMarkup(createElement(LocaleProvider, {
+ locale: 'en', children: createElement(TurnView, { turn }),
+ }));
+ const { document } = parseHTML(`<html><body>${markup}</body></html>`);
+ assert.equal(document.querySelectorAll('.maka-chat-message-bubble-assistant').length, 0);
+ assert.match(document.body.textContent, /partial thought/);
+ assert.match(document.body.textContent, /Response stream ended before completion/);
+});
+
test('renders steering where it arrived in the assistant timeline', () => {
const turn: TurnViewModel = {
turnId: 'turn-1',
diff --git a/packages/ui/src/__tests__/materialize.test.ts b/packages/ui/src/__tests__/materialize.test.ts
index 398eeede2e..15075a98af 100644
--- a/packages/ui/src/__tests__/materialize.test.ts
+++ b/packages/ui/src/__tests__/materialize.test.ts
@@ -19,7 +19,8 @@
import assert from "node:assert/strict";
import { describe, test } from "node:test";
-import type { StoredMessage } from '@maka/core/session';
+import { decodeCanonicalMessage, type StoredMessage } from '@maka/core/session';
+import { foldTimeline } from '../timeline-fold.js';
import {
materializeChat,
materializeTools,
@@ -37,6 +38,25 @@ const originalUser = {
ts: 1,
text: "request",
};
+
+test('keeps interrupted and replacement responses distinct live and after reload', () => {
+ const failed = { type: 'assistant', id: 'failed', turnId: 't1', ts: 2, text: 'Partial answer', modelId: 'fixture', interrupted: true };
+ const recovered = { ...failed, id: 'recovered', ts: 3, text: 'Recovered answer', interrupted: undefined };
+ const reloaded = JSON.parse(JSON.stringify([originalUser, failed, recovered])).map(decodeCanonicalMessage);
+ let live = armLiveTurn('t1');
+ for (const message of [failed, recovered]) {
+ live = applyLiveTurnEvent(live, {
+ type: 'text_complete', id: message.id, turnId: 't1', messageId: message.id,
+ ts: message.ts, text: message.text, ...(message.interrupted ? { interrupted: true } : {}),
+ })!;
+ }
+ for (const turn of [materializeTurns(reloaded, 'en')[0], overlayLiveTurn(materializeTurns([originalUser], 'en'), live, 'en')[0]]) {
+ const responses = foldTimeline(turn!.timeline).filter((entry) => entry.kind === 'text');
+ assert.deepEqual(responses.map((entry) => [entry.text, entry.interrupted === true]), [
+ ['Partial answer', true], ['Recovered answer', false],
+ ]);
+ }
+});
const beforeAssistant = {
type: "assistant" as const,
id: "before-steer",
diff --git a/packages/ui/src/chat-turn.tsx b/packages/ui/src/chat-turn.tsx
index d30d0fcfc6..5f7699ba0f 100644
--- a/packages/ui/src/chat-turn.tsx
+++ b/packages/ui/src/chat-turn.tsx
@@ -1159,10 +1159,11 @@ export function ModelProviderRetryIndicator(props: { retry: LiveProviderRetry })
type AssistantAnswerPhase = 'historical' | 'streaming' | 'settled';
type AssistantAnswerBubbleProps =
- | { text: string; phase: 'historical' }
+ | { text: string; phase: 'historical'; interrupted?: true }
| {
text: string;
phase: 'streaming' | 'settled';
+ interrupted?: true;
/** Text already streamed before this mount, so a remount does not replay it. */
settledText?: string;
truncated?: boolean;
@@ -1197,10 +1198,13 @@ const AssistantAnswerBubble = memo(function AssistantAnswerBubble(props: Assista
}, [props.phase, onSettled]);
return (
+ <>
+ {(!props.interrupted || props.text.length > 0) && (
<ChatMessageBubble
variant="ghost"
data-maka-transcript-boundary=""
data-live-streaming={props.phase === 'streaming' ? 'true' : undefined}
+ data-response-interrupted={props.interrupted ? 'true' : undefined}
// Astryx's own seam for a bubble that spans the message column: it sets
// the width and drops the default max(80%, 280px) cap in one prop.
width="100%"
@@ -1247,6 +1251,13 @@ const AssistantAnswerBubble = memo(function AssistantAnswerBubble(props: Assista
</Tooltip>
)}
</ChatMessageBubble>
+ )}
+ {props.interrupted && (
+ <ChatSystemMessage variant="divider">
+ {copy.providerRetryReason.stream_truncated}
+ </ChatSystemMessage>
+ )}
+ </>
);
});
@@ -1287,10 +1298,11 @@ function TurnTimelineEntry(props: {
);
}
// Same component either way — a type swap here would remount the answer.
- if (item.live !== true) return <AssistantAnswerBubble text={item.text} phase="historical" />;
+ if (item.live !== true) return <AssistantAnswerBubble text={item.text} interrupted={item.interrupted} phase="historical" />;
return (
<AssistantAnswerBubble
text={item.text}
+ interrupted={item.interrupted}
phase={item.complete === true ? 'settled' : 'streaming'}
settledText={props.initialLiveContent?.get(`text:${item.messageId}`)}
truncated={item.truncated === true}
diff --git a/packages/ui/src/live-turn-projection.ts b/packages/ui/src/live-turn-projection.ts
index bd48666543..5231e91ed6 100644
--- a/packages/ui/src/live-turn-projection.ts
+++ b/packages/ui/src/live-turn-projection.ts
@@ -74,6 +74,7 @@ export interface LiveTurnStepProjection {
export type LiveTurnStepContentKind = 'thinking' | 'text' | 'tools';
export interface LiveTextProjection {
+ interrupted?: true;
text: string;
truncated: boolean;
complete: boolean;
@@ -361,6 +362,7 @@ export function applyLiveTurnEvent(
nextStep = {
...step,
text: {
+ ...(event.interrupted ? { interrupted: true } : {}),
text: applied.text,
truncated: applied.truncated,
complete: true,
diff --git a/packages/ui/src/materialize.ts b/packages/ui/src/materialize.ts
index 23262feedb..8ebc7caa7f 100644
--- a/packages/ui/src/materialize.ts
+++ b/packages/ui/src/materialize.ts
@@ -382,6 +382,7 @@ export type TurnTimelineItem =
}
| {
kind: "text";
+ interrupted?: true;
text: string;
messageId: string;
ts?: number;
@@ -590,10 +591,11 @@ export function overlayLiveTurn(
live: step.thinking.complete !== true,
truncated: step.thinking.truncated,
});
- } else if (kind === "text" && step.text?.text) {
+ } else if (kind === "text" && step.text && (step.text.text || step.text.interrupted)) {
liveTimeline.push({
kind: "text",
text: step.text.text,
+ ...(step.text.interrupted ? { interrupted: true } : {}),
messageId: step.stepId,
live: true,
complete: step.text.complete,
@@ -1060,7 +1062,8 @@ export function projectTurnTools(
* - leftover buffered tools (abort / pure-tool turn with no assistant row)
* flush as a trailing tools group.
*
- * Empty text/thinking produce no item. Adjacent thinking blocks merge with
+ * Empty text/thinking produce no item, except text carrying an interruption
+ * divider. Adjacent thinking blocks merge with
* a blank line; adjacent tools groups merge into one group.
*/
function buildTurnTimeline(
@@ -1121,10 +1124,11 @@ function buildTurnTimeline(
text: message.thinking.text,
messageId: rowId,
});
- } else if (kind === "text" && message.text.length > 0) {
+ } else if (kind === "text" && (message.text.length > 0 || message.interrupted)) {
raw.push({
kind: "text",
text: message.text,
+ ...(message.interrupted ? { interrupted: true } : {}),
messageId: rowId,
ts: message.ts,
});
@@ -1144,10 +1148,11 @@ function buildTurnTimeline(
});
}
flushTools(legacy);
- if (message.text.length > 0) {
+ if (message.text.length > 0 || message.interrupted) {
raw.push({
kind: "text",
text: message.text,
+ ...(message.interrupted ? { interrupted: true } : {}),
messageId: rowId,
ts: message.ts,
});
diff --git a/packages/ui/src/timeline-fold.ts b/packages/ui/src/timeline-fold.ts
index b8be037bb4..f69e48ef28 100644
--- a/packages/ui/src/timeline-fold.ts
+++ b/packages/ui/src/timeline-fold.ts
@@ -57,6 +57,10 @@ export function foldTimeline(items: readonly TurnTimelineItem[]): FoldedTimeline
flush();
out.push(item);
anchor = item.messageId;
+ } else if (item.kind === 'text' && item.interrupted) {
+ buffer.push(item);
+ flush();
+ anchor = item.messageId;
} else {
buffer.push(item);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org