You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maka.apache.org by as...@apache.org on 2026/09/01 14:19:31 UTC
(maka) 01/09: feat(core): represent video and pdf model modalities
This is an automated email from the ASF dual-hosted git repository.
Astro-Han pushed a commit to branch ci/4398-model-metadata-drift-check
in repository https://gitbox.apache.org/repos/asf/maka.git
commit 5d9f01c66bc61a8ca73c817273d37bd5f6bb291f
Author: AstroHan <le...@outlook.com>
AuthorDate: Tue Sep 1 18:15:31 2026 +0800
feat(core): represent video and pdf model modalities
models.dev now declares video on either side of a model and pdf as an
output. The projector named four input and three output values and threw
on anything else, so 324 of the 1906 models in the selected providers
took the whole refresh down with them: `npm run refresh:model-metadata`
could not write a byte, and the snapshot could not be regenerated at all.
Admit both values across the three places that name them — the wire type,
the model-facts guard, and the catalog decoder — so a refresh reaches the
models that carry them. Both directions now accept the same set, so the
two decoders collapse into one that takes the direction; keeping two
copies only lets one drift behind the catalog it decodes.
The decoder rejects any value it does not name, so a newer Host
describing such a model fails an older client's catalog decode outright
rather than losing one field. That is a compatibility break, hence the
epoch bump: the handshake keeps the pairing from forming.
The snapshot itself is untouched here. Its content refresh belongs to the
scheduled job's own review pull request, not to this change.
Generated-by: Claude Code
---
.../src/__tests__/runtime-policy-codec.test.ts | 13 +++++++++
packages/core/src/llm-connections.ts | 4 +--
packages/core/src/model-catalog.ts | 7 ++---
packages/core/src/model-facts.ts | 15 ++++++----
.../src/runtime-policy/connection-catalog-codec.ts | 32 ++++++++++++----------
packages/runtime-host/src/protocol/index.ts | 8 +++++-
scripts/sync-model-metadata.mjs | 12 ++++----
scripts/sync-model-metadata.test.mjs | 32 +++++++++++++++++++++-
8 files changed, 89 insertions(+), 34 deletions(-)
diff --git a/packages/core/src/__tests__/runtime-policy-codec.test.ts b/packages/core/src/__tests__/runtime-policy-codec.test.ts
index 06bdd1328c..d6aba85a60 100644
--- a/packages/core/src/__tests__/runtime-policy-codec.test.ts
+++ b/packages/core/src/__tests__/runtime-policy-codec.test.ts
@@ -511,6 +511,19 @@ test('normalizes extended model facts used by the runtime host catalog', () => {
});
});
+test('carries the video and pdf modalities models.dev declares', () => {
+ const modalities = {
+ input: ['text', 'image', 'video'],
+ output: ['text', 'pdf', 'video'],
+ };
+ const result = normalizeConnectionModelDiscoveryResult({
+ models: [{ id: 'custom-model', modalities }],
+ source: 'fetched',
+ fetchedAt: 42,
+ });
+ assert.deepEqual(result.models[0], { id: 'custom-model', modalities });
+});
+
test('rejects sparse model modality arrays', () => {
assert.throws(
() =>
diff --git a/packages/core/src/llm-connections.ts b/packages/core/src/llm-connections.ts
index 5a4a839058..232a3a60e3 100644
--- a/packages/core/src/llm-connections.ts
+++ b/packages/core/src/llm-connections.ts
@@ -116,8 +116,8 @@ export interface ModelInfo {
};
/** Multimodal input/output support from provider catalog metadata. */
modalities?: {
- input: Array<'text' | 'image' | 'audio' | 'pdf'>;
- output: Array<'text' | 'image' | 'audio'>;
+ input: Array<'text' | 'image' | 'audio' | 'pdf' | 'video'>;
+ output: Array<'text' | 'image' | 'audio' | 'pdf' | 'video'>;
};
/**
* Read-time provenance for values overlaid from model-facts.json. This is
diff --git a/packages/core/src/model-catalog.ts b/packages/core/src/model-catalog.ts
index 12d2609ac4..45e5db3c0d 100644
--- a/packages/core/src/model-catalog.ts
+++ b/packages/core/src/model-catalog.ts
@@ -515,10 +515,9 @@ function displayNameForKnownModel(
* never set `capabilities.imageGeneration` for any of them, so the capability
* check below could not fire on bundled data.
*
- * An EMPTY list is not evidence. `modalities.output` is typed to text, image,
- * and audio, so a video model's real output has no representation and
- * serializes as `[]` — the same shape a future generator bug would produce.
- * Only a non-empty list says something, and what it says is what it lists.
+ * An EMPTY list is not evidence. A provider that declared no output modality
+ * and a generator bug that dropped them produce the same shape. Only a
+ * non-empty list says something, and what it says is what it lists.
*/
function declaresNoTextOutput(model: ModelInfo): boolean {
const output = model.modalities?.output;
diff --git a/packages/core/src/model-facts.ts b/packages/core/src/model-facts.ts
index 99f9ecbbef..11336a5dfd 100644
--- a/packages/core/src/model-facts.ts
+++ b/packages/core/src/model-facts.ts
@@ -194,7 +194,7 @@ function normalizeModalities(value: unknown): NonNullable<ModelFactOverride['mod
if (value.input === undefined && value.output === undefined)
throw new Error('Invalid modalities');
const input = normalizeModalityDirection(value.input, isModality);
- const output = normalizeModalityDirection(value.output, isOutputModality);
+ const output = normalizeModalityDirection(value.output, isModality);
return {
...(input === undefined ? {} : { input }),
...(output === undefined ? {} : { output }),
@@ -212,11 +212,14 @@ function normalizeModalityDirection<T extends string>(
return [...new Set(entries)];
}
-function isModality(value: unknown): value is 'text' | 'image' | 'audio' | 'pdf' {
- return value === 'text' || value === 'image' || value === 'audio' || value === 'pdf';
-}
-function isOutputModality(value: unknown): value is 'text' | 'image' | 'audio' {
- return value === 'text' || value === 'image' || value === 'audio';
+function isModality(value: unknown): value is 'text' | 'image' | 'audio' | 'pdf' | 'video' {
+ return (
+ value === 'text' ||
+ value === 'image' ||
+ value === 'audio' ||
+ value === 'pdf' ||
+ value === 'video'
+ );
}
function isPositiveBoundedInteger(value: unknown): value is number {
return (
diff --git a/packages/core/src/runtime-policy/connection-catalog-codec.ts b/packages/core/src/runtime-policy/connection-catalog-codec.ts
index fa0c6a1c15..9292b69cfe 100644
--- a/packages/core/src/runtime-policy/connection-catalog-codec.ts
+++ b/packages/core/src/runtime-policy/connection-catalog-codec.ts
@@ -600,23 +600,27 @@ function decodeModelModalities(value: unknown): NonNullable<ConnectionModel['mod
if (!Array.isArray(item.input) || !Array.isArray(item.output)) {
throw domainError('connection model modalities must contain input and output arrays');
}
- const input = Array.from(item.input, (entry) => decodeModelInputModality(entry));
- const output = Array.from(item.output, (entry) => decodeModelOutputModality(entry));
+ const input = Array.from(item.input, (entry) => decodeModelModality(entry, 'input'));
+ const output = Array.from(item.output, (entry) => decodeModelModality(entry, 'output'));
return { input, output };
}
-function decodeModelInputModality(value: unknown): 'text' | 'image' | 'audio' | 'pdf' {
- const modality = stringValue(value, 'connection model input modality', 16);
- if (modality !== 'text' && modality !== 'image' && modality !== 'audio' && modality !== 'pdf') {
- throw domainError('connection model input modality is invalid');
- }
- return modality;
-}
-
-function decodeModelOutputModality(value: unknown): 'text' | 'image' | 'audio' {
- const modality = stringValue(value, 'connection model output modality', 16);
- if (modality !== 'text' && modality !== 'image' && modality !== 'audio') {
- throw domainError('connection model output modality is invalid');
+// Both directions accept the same set. models.dev declares video on either
+// side and pdf on both, so splitting them again only invites one direction to
+// drift behind the catalog it decodes.
+function decodeModelModality(
+ value: unknown,
+ direction: 'input' | 'output',
+): 'text' | 'image' | 'audio' | 'pdf' | 'video' {
+ const modality = stringValue(value, `connection model ${direction} modality`, 16);
+ if (
+ modality !== 'text' &&
+ modality !== 'image' &&
+ modality !== 'audio' &&
+ modality !== 'pdf' &&
+ modality !== 'video'
+ ) {
+ throw domainError(`connection model ${direction} modality is invalid`);
}
return modality;
}
diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts
index 8aff63a16a..0a7784ff03 100644
--- a/packages/runtime-host/src/protocol/index.ts
+++ b/packages/runtime-host/src/protocol/index.ts
@@ -95,7 +95,13 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const;
export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const;
// Increment when the same protocol version no longer guarantees safe Client-Host
// interoperability. Mismatches are rejected before domain commands are admitted.
-export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 87 as const;
+export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 88 as const;
+// 88: Catalog model modalities admit video on either side and pdf as output.
+// models.dev declares both, and the modality decoder rejects any value it does
+// not name, so a newer Host describing such a model fails an older client's
+// catalog decode outright rather than losing one field. The handshake keeps
+// that pairing from forming; a newer client simply never sees the new values
+// from an older Host.
// 87: The connection catalog projects each model as the Host resolved it —
// a `catalog_entry` item per model, counted by the connection header. Clients
// render those entries instead of merging the stored row against their own
diff --git a/scripts/sync-model-metadata.mjs b/scripts/sync-model-metadata.mjs
index 2869c04416..1a6496357a 100644
--- a/scripts/sync-model-metadata.mjs
+++ b/scripts/sync-model-metadata.mjs
@@ -25,6 +25,10 @@ import { dirname } from 'node:path';
import { pathToFileURL } from 'node:url';
const SOURCE_URL = 'https://models.dev/api.json';
+// Must stay in step with ModelInfo['modalities'] in packages/core. A value
+// that reaches the projection but not the wire type fails the build; a value
+// missing here drops the whole refresh, not just the model that declares it.
+const MODALITIES = new Set(['text', 'image', 'audio', 'pdf', 'video']);
const DEFAULT_SNAPSHOT = 'scripts/model-metadata/models-dev-api.snapshot.json';
const DEFAULT_OUTPUT = 'packages/core/src/model-metadata.generated.ts';
const DEFAULT_PRICING_OUTPUT = 'packages/runtime/src/telemetry/model-pricing.generated.ts';
@@ -540,12 +544,8 @@ export function toMetadata(providerId, modelId, provider, model) {
throw new Error(`models.dev model ${providerId}/${modelId} has an unsupported shape`);
}
if (
- model.modalities?.input.some(
- (value) => value !== 'text' && value !== 'image' && value !== 'audio' && value !== 'pdf',
- ) ||
- model.modalities?.output.some(
- (value) => value !== 'text' && value !== 'image' && value !== 'audio',
- )
+ model.modalities?.input.some((value) => !MODALITIES.has(value)) ||
+ model.modalities?.output.some((value) => !MODALITIES.has(value))
) {
throw new Error(`models.dev model ${providerId}/${modelId} has unsupported modalities`);
}
diff --git a/scripts/sync-model-metadata.test.mjs b/scripts/sync-model-metadata.test.mjs
index 021b2aaf85..b0c9f0eef3 100644
--- a/scripts/sync-model-metadata.test.mjs
+++ b/scripts/sync-model-metadata.test.mjs
@@ -280,7 +280,7 @@ test('refresh rejects unknown model modalities instead of dropping them', async
const snapshot = join(root, 'snapshot.json');
const metadata = join(root, 'metadata.ts');
const catalog = fixtureCatalog();
- catalog.anthropic.models.model.modalities = { input: ['text', 'video'], output: ['text'] };
+ catalog.anthropic.models.model.modalities = { input: ['text', 'hologram'], output: ['text'] };
await writeFile(input, JSON.stringify(catalog));
await assert.rejects(
@@ -302,6 +302,36 @@ test('refresh rejects unknown model modalities instead of dropping them', async
}
});
+test('refresh carries the video and pdf modalities models.dev declares', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'maka-model-snapshot-video-'));
+ try {
+ const input = join(root, 'api.json');
+ const snapshot = join(root, 'snapshot.json');
+ const metadata = join(root, 'metadata.ts');
+ const catalog = fixtureCatalog();
+ const modalities = { input: ['text', 'video'], output: ['text', 'pdf', 'video'] };
+ catalog.anthropic.models.model.modalities = modalities;
+ await writeFile(input, JSON.stringify(catalog));
+
+ await main([
+ 'node',
+ 'sync-model-metadata.mjs',
+ '--refresh',
+ '--refresh-input',
+ input,
+ '--snapshot',
+ snapshot,
+ '--output',
+ metadata,
+ ]);
+
+ const written = JSON.parse(await readFile(snapshot, 'utf8'));
+ assert.deepEqual(written.projection.metadata.anthropic.model.modalities, modalities);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
test('refresh rejects a partial provider shrink until it is explicitly accepted', async () => {
const root = await mkdtemp(join(tmpdir(), 'maka-model-snapshot-provider-shrink-'));
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org