You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maka.apache.org by as...@apache.org on 2026/09/05 18:09:56 UTC
(maka) 01/01: fix(storage): reclaim the capture files the upgrade orphaned
This is an automated email from the ASF dual-hosted git repository.
Astro-Han pushed a commit to branch fix/4808-reclaim-capture-residue
in repository https://gitbox.apache.org/repos/asf/maka.git
commit 73cd9eb918b64bea1906bd8168a4bacca4a234c5
Author: AstroHan <le...@outlook.com>
AuthorDate: Sun Sep 6 02:09:10 2026 +0800
fix(storage): reclaim the capture files the upgrade orphaned
The v1 upgrade rebuilds the artifact catalog by dropping the table and
re-inserting what still decodes. Prepared-request captures name a source
this build no longer has, so their records go and their bytes stay --
unreferenced, and unreachable from the catalog that used to name them.
The store now deletes them on recovery, by the file name the retired
capture writer generated. Reading the name rather than the catalog is
what makes this work on a store the upgrade converts now and on one an
earlier build already converted, and it costs one directory listing per
Session with nothing resolved or stat'd. When the names are gone the
listing finds nothing, which is the whole record of having finished.
Generated-by: Claude Code
---
.../src/server/execution-composition.ts | 9 ++++
.../storage/src/__tests__/artifact-store.test.ts | 48 +++++++++++++++++
packages/storage/src/artifact-store.ts | 62 +++++++++++++++++++++-
packages/storage/src/artifact-stores.ts | 2 +
4 files changed, 120 insertions(+), 1 deletion(-)
diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts
index d6fad15d56..3e79c31bb7 100644
--- a/packages/runtime-host/src/server/execution-composition.ts
+++ b/packages/runtime-host/src/server/execution-composition.ts
@@ -1749,6 +1749,15 @@ export async function createExecutionRuntimeHostComposition(
recovery: {
state: async () => {
await skills.recover();
+ try {
+ await openedArtifactStore.reclaimRetiredCaptureFiles();
+ } catch (error) {
+ // Reclaiming leftover bytes is not worth refusing to start over.
+ // The next start lists the same names and tries again.
+ console.error(
+ `[runtime-host] retired provider-request captures could not be reclaimed: ${generalizedErrorMessage(error)}`,
+ );
+ }
},
},
drain: [
diff --git a/packages/storage/src/__tests__/artifact-store.test.ts b/packages/storage/src/__tests__/artifact-store.test.ts
index 6119b08a40..38b5119cfc 100644
--- a/packages/storage/src/__tests__/artifact-store.test.ts
+++ b/packages/storage/src/__tests__/artifact-store.test.ts
@@ -900,6 +900,54 @@ describe('SQLite Artifact store', () => {
});
});
+ test('reclaims the capture files the upgrade orphaned and nothing else', async () => {
+ await withWorkspace(async (root) => {
+ const store = createArtifactStore(root);
+ const kept = await store.create(artifactInput('kept', 'live bytes', 1));
+ // Named exactly like a capture, but a record still claims it.
+ const upload = await store.create({
+ ...artifactInput('upload', 'a user picked this name', 2),
+ name: 'provider-request-step-7-mine.json',
+ source: 'user_upload' as const,
+ });
+ const artifacts = join(root, 'artifacts');
+ const outside = join(root, 'outside');
+ await mkdir(join(artifacts, 'session-2'), { recursive: true });
+ await mkdir(outside, { recursive: true });
+ const orphans = [
+ 'session-1/f5ccbbca-7254-4743-bb65-97e549f49ec1-provider-request-step-0-1ea040d1-35fe-4c7e-a686-5e898ca79a45.json',
+ 'session-2/e28e050a-d2e5-4c29-9c7a-c206c6d626c4-provider-request-step-12-821bec76-a555-4460-ae4d-b4d125e736d5.json',
+ ];
+ const strangers = [
+ 'session-1/notes.txt',
+ 'session-1/d10a4385-provider-request-step-none-7a990696.json',
+ 'session-1/provider-request-step-3-7a990696.json',
+ ];
+ for (const relativePath of [...orphans, ...strangers]) {
+ await writeFile(join(artifacts, relativePath), 'residue');
+ }
+ const beyond = join(outside, 'a-provider-request-step-1-b.json');
+ await writeFile(beyond, 'not ours');
+ await symlink(outside, join(artifacts, 'session-3'));
+
+ await store.reclaimRetiredCaptureFiles();
+ await store.reclaimRetiredCaptureFiles();
+
+ for (const relativePath of orphans) {
+ await assert.rejects(() => stat(join(artifacts, relativePath)), { code: 'ENOENT' });
+ }
+ for (const relativePath of strangers) {
+ assert.equal((await stat(join(artifacts, relativePath))).isFile(), true);
+ }
+ assert.equal((await stat(beyond)).isFile(), true);
+ assert.deepEqual(await readArtifactText(store, kept.id), { ok: true, text: 'live bytes' });
+ assert.deepEqual(await readArtifactText(store, upload.id), {
+ ok: true,
+ text: 'a user picked this name',
+ });
+ });
+ });
+
test('a partial purge retains its cleanup obligations across reopen', async (t) => {
await withWorkspace(async (root) => {
const authority = createArtifactStoreWriteAuthority(root);
diff --git a/packages/storage/src/artifact-store.ts b/packages/storage/src/artifact-store.ts
index c2c5519f04..db6a7ed97f 100644
--- a/packages/storage/src/artifact-store.ts
+++ b/packages/storage/src/artifact-store.ts
@@ -18,7 +18,7 @@
*/
import { createHash, randomUUID } from 'node:crypto';
-import { type BigIntStats, constants as fsConstants } from 'node:fs';
+import { type BigIntStats, type Dirent, constants as fsConstants } from 'node:fs';
import {
access,
copyFile,
@@ -26,6 +26,7 @@ import {
mkdir,
open,
readFile,
+ readdir,
realpath,
rm,
stat,
@@ -72,6 +73,13 @@ export const ARTIFACT_TEXT_PREVIEW_LIMIT_BYTES = 10 * 1024 * 1024;
export const ARTIFACT_BINARY_PREVIEW_LIMIT_BYTES = 50 * 1024 * 1024;
const ARTIFACT_PURGE_RESOLVE_CONCURRENCY = 8;
+/**
+ * The file name the retired prepared-request capture writer generated:
+ * `<artifactId>-provider-request-step-<step>-<captureId>.json`. Nothing has
+ * produced one since that writer was removed, and no user picked one.
+ */
+const RETIRED_CAPTURE_FILE_NAME = /^[A-Za-z0-9_-]+-provider-request-step-\d+-[A-Za-z0-9_-]+\.json$/;
+
interface ArtifactSessionSnapshot {
readonly records: readonly ArtifactRecord[];
readonly revision: ArtifactListRevision;
@@ -182,6 +190,7 @@ export interface ArtifactAuthorityStore extends DurableArtifactAttachmentReader
input: ConversationArtifactCopyInput,
): Promise<ConversationArtifactCopyResult>;
purgeSessionArtifacts(sessionId: string): Promise<void>;
+ reclaimRetiredCaptureFiles(): Promise<void>;
deleteOwnedArtifactInSession(
sessionId: string,
artifactId: string,
@@ -473,6 +482,48 @@ class SqliteArtifactStore implements ArtifactAuthorityStore {
});
}
+ /**
+ * Deletes the prepared-request capture files that no record names any more.
+ *
+ * The upgrade that dropped their records could not take their bytes with it:
+ * it runs inside the database transaction, where a rollback would leave the
+ * files gone for good. So the bytes outlive the catalog, and after the drop
+ * nothing in it can enumerate them.
+ *
+ * What identifies one is the name the retired writer generated, so this reads
+ * the same on a store the upgrade just converted and on one converted by an
+ * earlier build. Nothing is resolved or stat'd -- one listing per Session
+ * directory, then an unlink per matching name -- and symlinks are skipped
+ * rather than followed, which is why this cannot become the cold-start
+ * syscall storm that walking the tree once was. Once the names are gone the
+ * listing finds nothing, and that is the whole record of having finished:
+ * there is no marker to keep in step with the disk. A path some record still
+ * claims keeps its bytes, so an upload that happens to be named this way
+ * survives.
+ */
+ async reclaimRetiredCaptureFiles(): Promise<void> {
+ await this.enqueueMutation(async () => {
+ await this.prepareMutationUnlocked();
+ const claimed = new Set(this.records.map((record) => record.relativePath));
+ for (const sessionEntry of await listArtifactDirectory(this.artifactRoot)) {
+ if (!sessionEntry.isDirectory()) continue;
+ const directory = join(this.artifactRoot, sessionEntry.name);
+ const entries = await listArtifactDirectory(directory);
+ let touched = false;
+ try {
+ for (const entry of entries) {
+ if (!entry.isFile() || !RETIRED_CAPTURE_FILE_NAME.test(entry.name)) continue;
+ if (claimed.has(`${sessionEntry.name}/${entry.name}`)) continue;
+ touched = true;
+ await rm(join(directory, entry.name), { force: true });
+ }
+ } finally {
+ if (touched) await syncDirectory(directory);
+ }
+ }
+ });
+ }
+
private async replayExistingArtifactUnlocked(
existing: ArtifactRecord,
input: CreateArtifactInput,
@@ -1100,6 +1151,15 @@ async function assertArtifactDirectory(artifactRoot: string, directory: string):
}
}
+async function listArtifactDirectory(path: string): Promise<Dirent[]> {
+ try {
+ return await readdir(path, { withFileTypes: true });
+ } catch (error) {
+ if (isNotFound(error)) return [];
+ throw error;
+ }
+}
+
async function ensureRealDirectory(path: string): Promise<string> {
await access(path, fsConstants.R_OK);
return realpath(path);
diff --git a/packages/storage/src/artifact-stores.ts b/packages/storage/src/artifact-stores.ts
index 58d82b3d73..e6c1cfbded 100644
--- a/packages/storage/src/artifact-stores.ts
+++ b/packages/storage/src/artifact-stores.ts
@@ -72,6 +72,7 @@ export interface InteractiveArtifactStoreWriter extends DurableArtifactAttachmen
input: ConversationArtifactCopyInput,
): Promise<ConversationArtifactCopyResult>;
purgeSessionArtifacts(sessionId: string): Promise<void>;
+ reclaimRetiredCaptureFiles(): Promise<void>;
listPage: ArtifactAuthorityStore['listPage'];
listTurnArtifacts: ArtifactAuthorityStore['listTurnArtifacts'];
getInSession: ArtifactAuthorityStore['getInSession'];
@@ -177,6 +178,7 @@ function createWriterFacade(
return run(() => store.copyConversationArtifacts(acceptedInput));
},
purgeSessionArtifacts: (sessionId) => run(() => store.purgeSessionArtifacts(sessionId)),
+ reclaimRetiredCaptureFiles: () => run(() => store.reclaimRetiredCaptureFiles()),
deleteUserArtifactInSession: (sessionId, artifactId) =>
run(() => store.deleteUserArtifactInSession(sessionId, artifactId)),
close: () => {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org