You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maka.apache.org by as...@apache.org on 2026/09/15 00:05:08 UTC
(maka) branch fix/runtime-overflow-recompaction-per-step created (now 61b82c130a)
This is an automated email from the ASF dual-hosted git repository.
Astro-Han pushed a change to branch fix/runtime-overflow-recompaction-per-step
in repository https://gitbox.apache.org/repos/asf/maka.git
at 61b82c130a fix(runtime): budget context compaction per accepted step, not per send
This branch includes the following new commits:
new 61b82c130a fix(runtime): budget context compaction per accepted step, not per send
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org
(maka) 01/01: fix(runtime): budget context compaction per accepted step, not per send
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Astro-Han pushed a commit to branch fix/runtime-overflow-recompaction-per-step
in repository https://gitbox.apache.org/repos/asf/maka.git
commit 61b82c130a768928fd9a09880ae0164ee5c45136
Author: AstroHan <le...@outlook.com>
AuthorDate: Tue Sep 15 08:05:00 2026 +0800
fix(runtime): budget context compaction per accepted step, not per send
A long turn that overflowed once, folded, and then kept working (ten image
reads in the archived CAD run) overflowed again and died: the compaction
module was gated by `compactionAttemptedThisSend`, a boolean spent by the
first fold for the rest of the Turn. The bound that latch was introduced
for (#4559, #4653) is the compact-and-resend spin with no progress in
between, not the second overflow of a request the provider has since
grown by accepting new steps.
The budget is now keyed on the logical step: `compactionAttemptedStep`
records the step whose request last entered the module, and both the
proactive projection and reactive recovery refuse a second entry on that
same step. A step only advances when the provider accepts a request, so
every further entry stands on real progress; the total stays bounded by
maxSteps and by each step's physical request budget, and the summarizer
failure circuit is still latched for the whole Turn. Image omission spends
the same step budget and merges into the omission map instead of replacing
it, and the separate permanent "no fold after an omission" gate is gone.
`compactionAppliedThisSend` collapses into `replacedStepNumber`, which the
reactive fold never set: an accepted retry after a reactive fold at step
N>0 carried a smaller input and was reported as `context_provider_dropping`.
The provider-dropping exclusion, the cross-send comparison and the
`context_overflow_after_compaction` note now all read that one field.
Generated-by: Claude Code
---
.../__tests__/mid-turn-capacity-backend.test.ts | 22 ++--
.../__tests__/overflow-reactive-recovery.test.ts | 114 +++++++++++++++++++--
packages/runtime/src/ai-sdk-compaction.ts | 73 ++++++-------
packages/runtime/src/ai-sdk-turn.ts | 18 ++--
4 files changed, 171 insertions(+), 56 deletions(-)
diff --git a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts
index 9238cdd0c6..75172cc11a 100644
--- a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts
+++ b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts
@@ -1138,15 +1138,25 @@ function defineMidTurnSuite(consumer: ConsumerMode): void {
assert.match(thirdPrompt, /tool_result_pruned/);
});
- test('compacts at most once per send', async () => {
- // The proactive fold already covers everything except the live head. A
- // second fold in the same send would buy the small verbatim tail and cost
- // the most recent context, so the send spends one call and no more.
+ test('compacts at most once per step, and again once a step is accepted', async () => {
+ // The budget is one fold per logical step, not one per send. The first
+ // fold covers everything except the live head, so re-entering on that same
+ // step would buy the small verbatim tail and cost the most recent context.
+ // The steps after it are different requests the provider accepted, and
+ // their own results put the baseline back over the window, so each earns
+ // its own fold instead of running out of remedies mid-turn.
const fixture = buildFixture({ priorChars: 2_000, rollingOverflow: true });
await runFixtureTurn(fixture, consumer);
- assert.equal(fixture.summarizerCalls, 1);
- assert.equal(fixture.recorded.length, 1);
+ assert.equal(fixture.summarizerCalls, 2);
+ assert.equal(fixture.recorded.length, 2);
+ // The second fold covers the step the first one left out.
+ assert.equal(
+ fixture.recorded[1]!.coverage.eventCount > fixture.recorded[0]!.coverage.eventCount,
+ true,
+ );
+ const complete = fixture.events.find((event) => event.type === 'complete');
+ assert.equal(complete?.type === 'complete' ? complete.stopReason : undefined, 'end_turn');
});
test('a prune-rescuable step is rescued by the prune, not compacted (review finding C)', async () => {
diff --git a/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts b/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts
index 800b43d395..10f212d8f4 100644
--- a/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts
+++ b/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts
@@ -166,6 +166,8 @@ interface ReactiveFixtureOptions {
maxSteps?: number;
/** The FIRST tool step reports an unusable usage object (no token counts). */
firstStepUsageMissing?: boolean;
+ /** Per provider-call input tokens, keyed by 1-based call number. */
+ inputTokensByCall?: Record<number, number>;
/** Tool-search availability with the deferred `Big` tool. */
gatedToolGroup?: boolean;
/**
@@ -262,15 +264,19 @@ function buildReactiveFixture(options: ReactiveFixtureOptions): ReactiveFixture
usage:
options.firstStepUsageMissing && call === 1
? ({ inputTokens: {}, outputTokens: {} } as ReturnType<typeof usage>)
- : usage(100, 20),
+ : usage(options.inputTokensByCall?.[call] ?? 100, 20),
},
];
- const doneChunks = (): LanguageModelV4StreamPart[] => [
+ const doneChunks = (call: number): LanguageModelV4StreamPart[] => [
{ type: 'stream-start', warnings: [] },
{ type: 'text-start', id: 'text-1' },
{ type: 'text-delta', id: 'text-1', delta: 'done' },
{ type: 'text-end', id: 'text-1' },
- { type: 'finish', finishReason: { unified: 'stop', raw: 'stop' }, usage: usage(120, 10) },
+ {
+ type: 'finish',
+ finishReason: { unified: 'stop', raw: 'stop' },
+ usage: usage(options.inputTokensByCall?.[call] ?? 120, 10),
+ },
];
const streamForCall = (call: number): ReadableStream<LanguageModelV4StreamPart> => {
const kind = options.script[call - 1];
@@ -426,7 +432,7 @@ function buildReactiveFixture(options: ReactiveFixtureOptions): ReactiveFixture
? toolCallChunks(call, 'tool_search', { query: 'Big' })
: kind === 'gated'
? toolCallChunks(call, 'Big', { q: 'run' })
- : doneChunks();
+ : doneChunks(call);
return simulateReadableStream({ chunks, initialDelayInMs: null, chunkDelayInMs: null });
};
const model = new MockLanguageModelV4({
@@ -1226,6 +1232,29 @@ describe('reactive overflow recovery in the streaming backend', () => {
assert.equal(lastCall?.totalTokens, 250);
});
+ test('a fold-shrunk retry input is not reported as provider dropping', async () => {
+ // The retry of step 1 is the folded request, so its input is smaller than
+ // step 0's by construction. Blaming the provider for that would be wrong
+ // and it is a once-per-session note, so the real one could never be shown.
+ const fixture = buildReactiveFixture({
+ script: ['tool', 'overflow', 'done'],
+ bigPriors: true,
+ inputTokensByCall: { 3: 80 },
+ });
+ await runTurn(fixture);
+
+ assert.equal(complete(fixture)?.stopReason, 'end_turn');
+ assert.equal(fixture.recorded.length, 1);
+ assert.equal(
+ fixture.messages.some(
+ (message) =>
+ (message as { type?: string; kind?: string }).type === 'system_note' &&
+ (message as { kind?: string }).kind === 'context_provider_dropping',
+ ),
+ false,
+ );
+ });
+
test('drops hydrated images before the single overflow retry without rerunning tools', async () => {
const fixture = buildReactiveFixture({
script: ['tool', 'overflow', 'done'],
@@ -1252,6 +1281,8 @@ describe('reactive overflow recovery in the streaming backend', () => {
});
test('surfaces a retryable second overflow after the image-only retry without another loop', async () => {
+ // Dropping the images spent this step's attempt, so the resend's own
+ // overflow is terminal.
const fixture = buildReactiveFixture({
script: ['tool', 'overflow503', 'overflow503'],
imagePrior: true,
@@ -1288,6 +1319,43 @@ describe('reactive overflow recovery in the streaming backend', () => {
}
});
+ test('a later step escalates from the image omission to a fold, keeping the images omitted', async () => {
+ // The first overflow is answered by dropping images alone. A step later
+ // the provider rejects again with the images already gone, so the next
+ // attempt has to fold — and the fold must not put the images back.
+ const fixture = buildReactiveFixture({
+ script: ['tool', 'overflow', 'tool', 'overflow', 'done'],
+ imagePrior: true,
+ bigPriors: true,
+ });
+ await runTurn(fixture);
+
+ assert.equal(fixture.model.doStreamCalls.length, 5);
+ assert.equal(complete(fixture)?.stopReason, 'end_turn');
+ assert.equal(
+ fixture.events.some((event) => event.type === 'error'),
+ false,
+ );
+ assert.deepEqual(fixture.toolExecutions, ['one.md', 'one.md']);
+ assert.equal(fixture.recorded.length, 1);
+ assert.equal(fixture.summarizerCalls(), 1);
+ // No request after the first omission ever carries the image again. The
+ // two that still replay the raw prior span carry the placeholder in its
+ // place; the folded one covers that span by the checkpoint instead.
+ for (const call of fixture.model.doStreamCalls.slice(2)) {
+ assert.equal(JSON.stringify(call.prompt).includes('"mediaType":"image/png"'), false);
+ }
+ for (const call of fixture.model.doStreamCalls.slice(2, 4)) {
+ assert.match(
+ JSON.stringify(call.prompt),
+ /Image artifact \\"screenshot\.png\\" omitted after provider context overflow/,
+ );
+ }
+ const foldedPrompt = JSON.stringify(fixture.model.doStreamCalls[4]?.prompt);
+ assert.equal(foldedPrompt.includes('REACTIVE_SUMMARY_SENTINEL'), true);
+ assert.equal(foldedPrompt.includes('screenshot.png'), false);
+ });
+
test('keeps a step-0 overflow checkpoint projected after the retry returns a tool call', async () => {
// The first request overflows before any step completes. Recovery folds
// only prior history into a pre-turn checkpoint and retries with the
@@ -1832,8 +1900,8 @@ describe('reactive overflow recovery in the streaming backend', () => {
});
await runTurn(fixture);
- // The latch permits exactly one compact-and-retry; the retry's overflow is
- // terminal, not a third attempt.
+ // Both overflows are the same logical step, which has one attempt: the
+ // retry's overflow is terminal, not a third attempt.
assert.equal(fixture.model.doStreamCalls.length, 3);
assert.equal(complete(fixture)?.stopReason, 'error');
assert.equal(
@@ -1844,6 +1912,40 @@ describe('reactive overflow recovery in the streaming backend', () => {
assert.equal(fixture.llmCalls.at(-1)?.errorClass, 'context_overflow');
});
+ test('a later step that overflows again folds again after real progress', async () => {
+ // The budget is one attempt per logical step, not one per send. The first
+ // overflow folds and the retry succeeds; two more tool steps are accepted
+ // by the provider, and the overflow that follows them is a different step
+ // with new history behind it, so it gets its own fold instead of failing
+ // the turn.
+ const fixture = buildReactiveFixture({
+ script: ['tool', 'overflow', 'tool', 'tool', 'overflow', 'done'],
+ bigPriors: true,
+ });
+ await runTurn(fixture);
+
+ assert.equal(fixture.model.doStreamCalls.length, 6);
+ assert.equal(complete(fixture)?.stopReason, 'end_turn');
+ assert.equal(
+ fixture.events.some((event) => event.type === 'error'),
+ false,
+ );
+ assert.equal(fixture.recorded.length, 2);
+ assert.equal(fixture.summarizerCalls(), 2);
+ // No completed tool step was replayed across either retry.
+ assert.deepEqual(fixture.toolExecutions, ['one.md', 'one.md', 'one.md']);
+ // The second fold covers the steps accepted after the first one, so it is
+ // a fold of new history rather than a repeat of the same prefix.
+ assert.equal(
+ fixture.recorded[1]!.coverage.eventCount > fixture.recorded[0]!.coverage.eventCount,
+ true,
+ );
+ assert.notEqual(
+ fixture.recorded[1]!.coverage.through.runtimeEventId,
+ fixture.recorded[0]!.coverage.through.runtimeEventId,
+ );
+ });
+
test('no recovery seam means a context-length overflow ends as a real error', async () => {
const fixture = buildReactiveFixture({
script: ['tool', 'overflow'],
diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts
index 6c5004ee37..9875ca6f0f 100644
--- a/packages/runtime/src/ai-sdk-compaction.ts
+++ b/packages/runtime/src/ai-sdk-compaction.ts
@@ -835,7 +835,7 @@ export class AiSdkCompaction {
* the next request neither describes (tool results, user text, images
* appended since) is judged by the provider when the request goes out. This
* hook never terminates the turn: every failure fails open with a diagnostic
- * and the request is sent; a rejection is recovered by one reactive fold
+ * and the request is sent; a rejection is recovered by a reactive fold
* (#4559).
*/
public buildMidTurnCapacityCompactProjection(
@@ -913,10 +913,10 @@ export class AiSdkCompaction {
state.capacity !== undefined &&
state.baselineTokens !== undefined &&
state.baselineTokens + state.replyReserveTokens >= state.capacity;
- if (!overWindow || state.compactionAttemptedThisSend) {
+ if (!overWindow || state.compactionAttemptedStep === options.stepNumber) {
return keepProjection();
}
- state.compactionAttemptedThisSend = true;
+ state.compactionAttemptedStep = options.stepNumber;
const activeToolsForStep = options.resolveDispatch(options.activeTools).activeTools;
// Fold a safe completed prefix of the durable turn ledger into a
// replacement projection (validate → persist), shared with the reactive
@@ -939,7 +939,6 @@ export class AiSdkCompaction {
}
// The fold replaced the request; the baseline described the old one.
// The next accepted request is the first measurement of the new shape.
- state.compactionAppliedThisSend = true;
state.baselineTokens = undefined;
acceptedProjection = {
sourceSignatures: incomingMessages.map(modelMessageSignature),
@@ -964,8 +963,8 @@ export class AiSdkCompaction {
* messages — the compaction core shared by the proactive projection stage
* (issue #882 PR 1) and the reactive overflow recovery (PR 2). It waits for
* the seq-ack durability boundary, reads the ledger, plans the fold, then
- * validates (materializable ∧ smaller than the reference request ∧
- * replay-admissible) and persists BEFORE returning the replacement, so a
+ * validates (materializable ∧ replay-admissible) and persists BEFORE
+ * returning the replacement, so a
* recovery re-projection never re-injects a covered raw span. It only shapes:
* the pass/terminate verdict and the diagnostic emission are the caller's.
*/
@@ -1199,8 +1198,9 @@ export class AiSdkCompaction {
/**
* Reactive overflow recovery (issue #882 PR 2): the second line of defense.
* When a provider rejects a request with a context-length error, fold the
- * durable turn ledger once and resend once — a single compact-and-retry
- * latch (pi's `_overflowRecoveryAttempted`). Returns the compacted messages
+ * durable turn ledger and resend — once per logical step, so the resend's
+ * own rejection is terminal but a step the provider has since accepted earns
+ * a fresh attempt. Returns the compacted messages
* to resend, or undefined when recovery is impossible or already spent, in
* which case the caller surfaces the real provider error rather than a
* fabricated success or a locally synthesized verdict (the provider — not the
@@ -1223,7 +1223,7 @@ export class AiSdkCompaction {
abortSignal?: AbortSignal;
}): Promise<{ messages: ModelMessage[] } | undefined> {
const state = input.midTurnState;
- if (!state || state.compactionAttemptedThisSend || state.omittedImageToolResults.size > 0) {
+ if (!state || state.compactionAttemptedStep === input.stepNumber) {
return undefined;
}
if (this.modelAdapter.classifyError(input.error) !== 'context_overflow') return undefined;
@@ -1231,20 +1231,24 @@ export class AiSdkCompaction {
const eligibleImages = collectHistoricalImageToolResults(state.priorContentEvents);
const imageOmission = omitHistoricalImageToolResults(input.currentMessages, eligibleImages);
if (imageOmission.omittedParts > 0) {
- state.omittedImageToolResults = new Map(
- [...imageOmission.omittedToolCallIds].flatMap((toolCallId) => {
- const image = eligibleImages.get(toolCallId);
- return image ? [[toolCallId, image] as const] : [];
- }),
- );
+ // Dropping images is this step's one attempt: a second overflow on the
+ // same step is terminal, while a later step retries with folding.
+ state.compactionAttemptedStep = input.stepNumber;
+ for (const toolCallId of imageOmission.omittedToolCallIds) {
+ const image = eligibleImages.get(toolCallId);
+ // Earlier steps' omissions stay omitted: every later request is
+ // projected through this map, so losing an entry would put its image
+ // back into the very history the provider just rejected.
+ if (image) state.omittedImageToolResults.set(toolCallId, image);
+ }
state.baselineTokens = undefined;
return { messages: imageOmission.messages };
}
const phase = state.compactionPhase(input.stepNumber);
- // Entering the module spends the send's one attempt whether or not a fold
- // comes out of it; only a selected projection sets `applied`.
- state.compactionAttemptedThisSend = true;
+ // Entering the module spends this step's one attempt whether or not a fold
+ // comes out of it; only a selected projection sets `replacedStepNumber`.
+ state.compactionAttemptedStep = input.stepNumber;
const outcome = await this.compactActiveRequestHistory({
turnId: input.turnId,
phase,
@@ -1290,7 +1294,7 @@ export class AiSdkCompaction {
}),
);
// The fold replaced the request; the baseline described the rejected one.
- state.compactionAppliedThisSend = true;
+ state.replacedStepNumber = input.stepNumber;
state.baselineTokens = undefined;
return { messages: outcome.replacementMessages };
}
@@ -1384,9 +1388,12 @@ export class MidTurnCapacityCompactState {
/** Checkpoint accepted during this send; pins every later durable projection. */
projectionCheckpoint: HistoryCompactCheckpoint | undefined;
/**
- * Step whose request the capacity hook replaced. Semantic/active-full
- * compaction yields on that exact step so one step never runs two
- * summarizers or double-projects.
+ * Step whose request a fold replaced, proactively or after an overflow.
+ *
+ * That step's request is not the one its history would have produced: if the
+ * provider accepted it, a smaller input is explained by the fold rather than
+ * by provider-side dropping; if the provider rejected it, what was rejected
+ * is the already-folded request.
*/
replacedStepNumber: number | undefined;
/**
@@ -1401,21 +1408,17 @@ export class MidTurnCapacityCompactState {
/** Malformed summaries spend one bounded repair budget for this whole Turn. */
summarizerFailure: string | undefined;
/**
- * The compaction module has been entered in this send.
- *
- * One attempt per send, whatever its outcome: the summarizer's own failure
- * circuit already latches for the rest of the send, so a second entry would
- * dispatch nothing new. This is the budget, and only the budget (#4559).
- */
- compactionAttemptedThisSend = false;
- /**
- * A folded projection was actually selected in this send.
+ * Logical step whose request last entered the compaction module.
*
- * Distinct from the attempt: a fold that fails open leaves the request
- * carrying its full raw history, so nothing may be concluded from a later
- * rejection about what remains in it.
+ * One attempt per step, whatever its outcome: the retries of one step share
+ * its number, so a compact-and-resend that overflows again spends nothing
+ * new and the error is surfaced. A step only advances once the provider has
+ * accepted a request, so every later entry stands on real progress. The
+ * total stays bounded by maxSteps and by each step's physical request budget
+ * (MAX_PROVIDER_ATTEMPTS_PER_STEP); the summarizer's failure circuit is
+ * still latched once for the whole Turn (#4559, #4653).
*/
- compactionAppliedThisSend = false;
+ compactionAttemptedStep: number | undefined;
/**
* Input tokens of the last request a provider accepted before this send.
*
diff --git a/packages/runtime/src/ai-sdk-turn.ts b/packages/runtime/src/ai-sdk-turn.ts
index ab9f867ea8..edad67d47f 100644
--- a/packages/runtime/src/ai-sdk-turn.ts
+++ b/packages/runtime/src/ai-sdk-turn.ts
@@ -1838,19 +1838,19 @@ export class AiSdkTurn {
// input tokens across eight turns with nothing reported
// (#4623). The first request of a send therefore compares
// against the persisted anchor, which is route-validated
- // where it is read; a fold before that request would explain
- // a smaller input by itself, so it disables the comparison.
+ // where it is read.
const acrossSends = completedRequestIndex === 0;
const priorInput = acrossSends
- ? midTurnState?.compactionAppliedThisSend === true
- ? undefined
- : midTurnState?.priorAcceptedInputTokens
+ ? midTurnState?.priorAcceptedInputTokens
: lastStepInputTokens;
if (
!this.deps.session.contextProviderDroppingReported &&
!toolSchemaShrank &&
midTurnState &&
priorInput !== undefined &&
+ // A fold, a prune or an image omission on this very request
+ // explains a smaller input by itself, whether the request is
+ // the first of the send or a later step.
midTurnState.replacedStepNumber !== completedRequestIndex &&
pruneAppliedAtStep !== completedRequestIndex &&
midTurnState.omittedImageToolResults.size === 0 &&
@@ -2036,7 +2036,7 @@ export class AiSdkTurn {
continue;
}
// Window suggestion (#4559): the provider rejected a request and
- // no recovery is left — the one fold is spent, or there was no
+ // no recovery is left — this step's fold is spent, or there was no
// seam. The baseline is a proven-fit total (input + output of an
// accepted request), so it is a number the user can declare; the
// trigger is `>=`, so declaring exactly it folds before this
@@ -2058,8 +2058,8 @@ export class AiSdkTurn {
: {}),
});
}
- // A folded projection was selected in this send and the provider
- // still rejects the request. That is worth saying, because the
+ // A folded projection was selected for THIS step's request and
+ // the provider still rejects it. That is worth saying, because the
// usual remedy has already been applied; it is NOT proof that the
// new message alone is the cause, since what remains also carries
// the system prompt, the tool schemas, the summary and the recent
@@ -2069,7 +2069,7 @@ export class AiSdkTurn {
if (
!contextOverflowAfterCompactionNoteWritten &&
failure.kind === 'context_overflow' &&
- midTurnState?.compactionAppliedThisSend === true
+ midTurnState?.replacedStepNumber === runtimeSteps
) {
contextOverflowAfterCompactionNoteWritten = true;
await this.recordSystemNote('context_overflow_after_compaction', turnId);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org