You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maka.apache.org by as...@apache.org on 2026/09/16 05:40:00 UTC
(maka) branch fix/model-discovery-catalog-contract created (now c80c04cb0e)
This is an automated email from the ASF dual-hosted git repository.
Astro-Han pushed a change to branch fix/model-discovery-catalog-contract
in repository https://gitbox.apache.org/repos/asf/maka.git
at c80c04cb0e fix(runtime): unify model discovery catalog validation
This branch includes the following new commits:
new c80c04cb0e fix(runtime): unify model discovery catalog validation
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org
(maka) 01/01: fix(runtime): unify model discovery catalog validation
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Astro-Han pushed a commit to branch fix/model-discovery-catalog-contract
in repository https://gitbox.apache.org/repos/asf/maka.git
commit c80c04cb0e00f0d2c270e896caa08832a800ad43
Author: AstroHan <le...@outlook.com>
AuthorDate: Wed Sep 16 13:39:20 2026 +0800
fix(runtime): unify model discovery catalog validation
Generated-by: Codex
---
.../src/__tests__/runtime-policy-codec.test.ts | 42 +++-
packages/core/src/runtime-policy.ts | 1 +
.../src/runtime-policy/connection-catalog-codec.ts | 33 ++-
.../connection-effect-coordinator.test.ts | 73 +++++++
.../src/__tests__/model-discovery-fixture.ts | 33 +++
.../runtime/src/__tests__/model-fetcher.test.ts | 222 ++++++++++++++-------
.../src/__tests__/provider-conformance.test.ts | 9 +-
.../src/__tests__/provider-contract-matrix.test.ts | 6 +-
.../src/__tests__/provider-contract-overrides.ts | 14 +-
packages/runtime/src/model-fetcher.ts | 101 ++--------
10 files changed, 346 insertions(+), 188 deletions(-)
diff --git a/packages/core/src/__tests__/runtime-policy-codec.test.ts b/packages/core/src/__tests__/runtime-policy-codec.test.ts
index 7efd198ba1..0e5526d47a 100644
--- a/packages/core/src/__tests__/runtime-policy-codec.test.ts
+++ b/packages/core/src/__tests__/runtime-policy-codec.test.ts
@@ -506,11 +506,6 @@ test('normalizes exact bounded model discovery results', () => {
},
);
for (const invalid of [
- {
- models: [{ id: 'duplicate' }, { id: 'duplicate' }],
- source: 'fetched',
- fetchedAt: 42,
- },
{ models: [{ id: 'gpt-5' }], source: 'unknown', fetchedAt: 42 },
{ models: [{ id: 'gpt-5' }], source: 'fetched', fetchedAt: 42, rawBody: 'secret' },
]) {
@@ -521,6 +516,43 @@ test('normalizes exact bounded model discovery results', () => {
}
});
+test('discovery and persisted catalogs enforce the same model list contract', () => {
+ const catalog = (models: unknown) =>
+ decodeCanonicalConnectionCatalogEntry({
+ connectionId: '123e4567-e89b-42d3-a456-426614174000',
+ revision: 1,
+ slug: 'fireworks',
+ name: 'Fireworks',
+ providerType: 'fireworks-ai',
+ enabled: true,
+ enabledModelIds: [],
+ models,
+ modelSource: 'fetched',
+ modelsFetchedAt: 42,
+ });
+ const discovery = (models: unknown) =>
+ normalizeConnectionModelDiscoveryResult({
+ models,
+ source: 'fetched',
+ fetchedAt: 42,
+ });
+ for (const decode of [catalog, discovery]) {
+ assert.deepEqual(decode([{ id: 'known', contextWindow: 128000 }, { id: 'unknown' }]).models, [
+ { id: 'known', contextWindow: 128000 },
+ { id: 'unknown' },
+ ]);
+ for (const models of [
+ [{ id: 'duplicate' }, { id: 'duplicate' }],
+ Array.from({ length: 2049 }, (_, i) => ({ id: `model-${i}` })),
+ ...[0, -1, 1.5, Number.MAX_SAFE_INTEGER + 1].map((contextWindow) => [
+ { id: 'invalid', contextWindow },
+ ]),
+ ]) {
+ assert.throws(() => decode(models), RuntimePolicyDomainDecodeError);
+ }
+ }
+});
+
test('normalizes extended model facts used by the runtime host catalog', () => {
const result = normalizeConnectionModelDiscoveryResult({
models: [
diff --git a/packages/core/src/runtime-policy.ts b/packages/core/src/runtime-policy.ts
index dc4491bafb..72c6a1e51c 100644
--- a/packages/core/src/runtime-policy.ts
+++ b/packages/core/src/runtime-policy.ts
@@ -70,6 +70,7 @@ export {
decodeConnectionCredentialTarget,
decodeModelOverridesTable,
decodeConnectionModel,
+ decodeConnectionModels,
decodeConnectionName,
decodeConnectionSlug,
decodeConnectionTarget,
diff --git a/packages/core/src/runtime-policy/connection-catalog-codec.ts b/packages/core/src/runtime-policy/connection-catalog-codec.ts
index c8e2ddce86..813dce30d2 100644
--- a/packages/core/src/runtime-policy/connection-catalog-codec.ts
+++ b/packages/core/src/runtime-policy/connection-catalog-codec.ts
@@ -431,16 +431,7 @@ export function decodeCanonicalConnectionCatalogEntry(value: unknown): Connectio
? {}
: { requestBodyOverlay: item.requestBodyOverlay }),
});
- if (
- !Array.isArray(item.models) ||
- item.models.length > CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION
- ) {
- throw domainError('connection models must be a bounded array');
- }
- const models = item.models.map(decodeConnectionModel);
- if (new Set(models.map((model) => model.id)).size !== models.length) {
- throw domainError('connection model ids must be unique');
- }
+ const models = decodeConnectionModels(item.models);
if (
item.modelSource !== undefined &&
item.modelSource !== 'fetched' &&
@@ -722,20 +713,22 @@ export function decodeConnectionTestSummary(value: unknown): ConnectionTestSumma
};
}
+export function decodeConnectionModels(value: unknown): ConnectionModel[] {
+ if (!Array.isArray(value) || value.length > CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION) {
+ throw domainError('connection models must be a bounded array');
+ }
+ const models = value.map(decodeConnectionModel);
+ if (new Set(models.map((model) => model.id)).size !== models.length) {
+ throw domainError('connection model ids must be unique');
+ }
+ return models;
+}
+
export function normalizeConnectionModelDiscoveryResult(
value: unknown,
): ConnectionModelDiscoveryResult {
const item = exactRecord(value, 'model discovery result', ['models', 'source', 'fetchedAt']);
- if (
- !Array.isArray(item.models) ||
- item.models.length > CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION
- ) {
- throw domainError('model discovery models must be a bounded array');
- }
- const models = item.models.map(decodeConnectionModel);
- if (new Set(models.map((model) => model.id)).size !== models.length) {
- throw domainError('model discovery model ids must be unique');
- }
+ const models = decodeConnectionModels(item.models);
if (item.source !== 'fetched' && item.source !== 'fallback') {
throw domainError('model discovery source is invalid');
}
diff --git a/packages/runtime-host/src/__tests__/connection-effect-coordinator.test.ts b/packages/runtime-host/src/__tests__/connection-effect-coordinator.test.ts
index 420b2d964c..09c877a3f7 100644
--- a/packages/runtime-host/src/__tests__/connection-effect-coordinator.test.ts
+++ b/packages/runtime-host/src/__tests__/connection-effect-coordinator.test.ts
@@ -51,6 +51,79 @@ const context: ConnectionContext = {
acquireResidency: () => ({ release: () => undefined }),
};
+test('Fireworks onboarding keeps an unrelated model with unknown limits and persists the selected K3', async () => {
+ const base = await mkdtemp(join(tmpdir(), 'maka-fireworks-onboarding-'));
+ const capability = await resolveStorageRoot({
+ path: join(base, 'interactive'),
+ kind: 'interactive',
+ });
+ const k3 = 'accounts/fireworks/models/kimi-k3';
+ const other = 'accounts/fireworks/models/qwen3p8-max';
+ const requests: string[] = [];
+ try {
+ for (const reopen of [false, true]) {
+ const owner = await tryAcquireInteractiveRootOwner(capability);
+ assert.ok(owner);
+ try {
+ const stores = await openInteractiveRuntimePolicyStoresForWrite(owner.lease);
+ if (!reopen) {
+ const coordinator = new HostConnectionEffectCoordinator({
+ stores,
+ activation: new RuntimePolicyActivationGate(),
+ oauthCredentials: new HostOAuthExecutionAuthority(stores),
+ createTransport: () => ({
+ fetch: async (input) => {
+ const path = new URL(String(input)).pathname;
+ requests.push(path);
+ if (path === '/v1/accounts') return Response.json({ accounts: [] });
+ assert.equal(path, '/v1/accounts/fireworks/models');
+ return Response.json({
+ models: [
+ { name: k3, contextLength: 1048576 },
+ { name: other, contextLength: 0 },
+ ],
+ });
+ },
+ close: async () => undefined,
+ }),
+ });
+ try {
+ const saved = await coordinator.handlers['connection.onboarding.save'](
+ {
+ target: {
+ kind: 'create',
+ providerType: 'fireworks-ai',
+ slug: 'fireworks',
+ name: 'Fireworks',
+ },
+ apiKey: 'fixture-key',
+ baseUrl: null,
+ enabledModelIds: [k3],
+ },
+ context,
+ );
+ assertSaved(saved);
+ } finally {
+ await coordinator.close();
+ }
+ }
+ const catalog = await stores.connectionCatalog.getSnapshot();
+ assert.equal(catalog.connections.length, 1);
+ assert.deepEqual(catalog.connections[0]?.enabledModelIds, [k3]);
+ assert.deepEqual(catalog.connections[0]?.models, [
+ { id: k3, contextWindow: 1048576 },
+ { id: other },
+ ]);
+ } finally {
+ await owner.close();
+ }
+ }
+ assert.deepEqual(requests, ['/v1/accounts', '/v1/accounts/fireworks/models']);
+ } finally {
+ await rm(base, { recursive: true, force: true });
+ }
+});
+
test('verifies a first-run API key without persisting a connection or credential', async () => {
await withFixture(async ({ stores }) => {
let observed: { slug: string; secret: string } | undefined;
diff --git a/packages/runtime/src/__tests__/model-discovery-fixture.ts b/packages/runtime/src/__tests__/model-discovery-fixture.ts
new file mode 100644
index 0000000000..fc09702926
--- /dev/null
+++ b/packages/runtime/src/__tests__/model-discovery-fixture.ts
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import assert from 'node:assert/strict';
+import type { LlmConnection, ModelInfo } from '@maka/core/llm-connections';
+import { runConnectionModelDiscoveryEffect } from '../model-fetcher.js';
+
+export async function discoverModels(
+ connection: LlmConnection,
+ apiKey: string,
+): Promise<ModelInfo[]> {
+ const outcome = await runConnectionModelDiscoveryEffect(connection, apiKey, {
+ fetch: globalThis.fetch,
+ });
+ if (!outcome.ok) assert.fail(outcome.error.kind);
+ return [...outcome.models];
+}
diff --git a/packages/runtime/src/__tests__/model-fetcher.test.ts b/packages/runtime/src/__tests__/model-fetcher.test.ts
index 6e615175a6..3ab1bdcc23 100644
--- a/packages/runtime/src/__tests__/model-fetcher.test.ts
+++ b/packages/runtime/src/__tests__/model-fetcher.test.ts
@@ -21,11 +21,8 @@ import assert from 'node:assert/strict';
import { createServer, type IncomingMessage, type ServerResponse } from 'node:http';
import { after, describe, test } from 'node:test';
import type { LlmConnection } from '@maka/core/llm-connections';
-import {
- fetchProviderModels,
- ProviderModelDiscoveryHttpError,
- runConnectionModelDiscoveryEffect,
-} from '../model-fetcher.js';
+import { runConnectionModelDiscoveryEffect } from '../model-fetcher.js';
+import { discoverModels } from './model-discovery-fixture.js';
const servers: Array<{ close(): Promise<void> }> = [];
@@ -33,7 +30,94 @@ after(async () => {
await Promise.all(servers.map((server) => server.close()));
});
-describe('fetchProviderModels', () => {
+describe('model discovery', () => {
+ test('all token metadata adapters preserve valid limits and omit invalid optional limits', async () => {
+ const values = [
+ undefined,
+ null,
+ '128000',
+ 0,
+ -1,
+ 1.5,
+ Number.MAX_SAFE_INTEGER + 1,
+ 1,
+ 128000,
+ Number.MAX_SAFE_INTEGER,
+ ];
+ const cases = [
+ {
+ providerType: 'fireworks-ai',
+ row: (id: string, value: unknown) => ({ name: id, contextLength: value }),
+ key: 'models',
+ output: false,
+ },
+ {
+ providerType: 'cohere',
+ row: (id: string, value: unknown) => ({
+ name: id,
+ context_length: value,
+ endpoints: ['chat'],
+ }),
+ key: 'models',
+ output: false,
+ },
+ {
+ providerType: 'openai-compatible',
+ row: (id: string, value: unknown) => ({ id, context_length: value, max_tokens: value }),
+ key: 'data',
+ output: true,
+ },
+ {
+ providerType: 'github-copilot',
+ row: (id: string, value: unknown) => ({
+ id,
+ model_picker_enabled: true,
+ supported_endpoints: ['/responses'],
+ capabilities: {
+ supports: { tool_calls: true },
+ limits: { max_context_window_tokens: value, max_output_tokens: value },
+ },
+ }),
+ key: 'data',
+ output: true,
+ },
+ {
+ providerType: 'openai-codex',
+ row: (id: string, value: unknown) => ({ slug: id, context_window: value }),
+ key: 'models',
+ output: false,
+ },
+ ] as const;
+ for (const fixture of cases) {
+ const outcome = await runConnectionModelDiscoveryEffect(
+ { providerType: fixture.providerType, baseUrl: 'https://fixture.invalid/v1' },
+ 'fixture-key',
+ {
+ fetch: async (input) =>
+ Response.json(
+ new URL(String(input)).pathname === '/v1/accounts'
+ ? { accounts: [] }
+ : { [fixture.key]: values.map((value, i) => fixture.row(`model-${i}`, value)) },
+ ),
+ },
+ );
+ assert.ok(outcome.ok, fixture.providerType);
+ assert.deepEqual(
+ outcome.models.map(({ id, contextWindow, maxOutputTokens }) => ({
+ id,
+ contextWindow,
+ maxOutputTokens,
+ })),
+ values.map((_, i) => ({
+ id: `model-${i}`,
+ contextWindow: i < 7 ? undefined : values[i],
+ maxOutputTokens: fixture.output && i >= 7 ? values[i] : undefined,
+ })),
+ fixture.providerType,
+ );
+ }
+ });
+
test('Cloudflare Workers AI accepts exactly 2,048 models and rejects the first excess item', async () => {
for (const modelCount of [2_048, 2_049]) {
let requestCount = 0;
@@ -50,24 +134,22 @@ describe('fetchProviderModels', () => {
respondJson(response, 200, { success: true, result });
});
- const request = fetchProviderModels(
+ const request = runConnectionModelDiscoveryEffect(
{
- slug: 'cloudflare-workers-ai',
- name: 'Cloudflare Workers AI',
providerType: 'cloudflare-workers-ai',
baseUrl: `${server.url}/client/v4/accounts/account-123/ai/v1`,
defaultModel: '@cf/example/default',
- enabled: true,
- createdAt: 1,
- updatedAt: 1,
},
'cloudflare-api-token',
+ { fetch: globalThis.fetch },
);
if (modelCount === 2_048) {
- assert.equal((await request).length, 2_048);
+ const outcome = await request;
+ assert.ok(outcome.ok);
+ assert.equal(outcome.models.length, 2_048);
assert.equal(requestCount, 42);
} else {
- await assert.rejects(request, /Failed to fetch provider models/);
+ assert.deepEqual(await request, { ok: false, error: { kind: 'invalid_response' } });
assert.equal(requestCount, 41);
}
}
@@ -91,21 +173,17 @@ describe('fetchProviderModels', () => {
});
});
- await assert.rejects(
- fetchProviderModels(
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(
{
- slug: 'cloudflare-workers-ai',
- name: 'Cloudflare Workers AI',
providerType: 'cloudflare-workers-ai',
baseUrl: `${server.url}/client/v4/accounts/account-123/ai/v1`,
defaultModel: '@cf/example/default',
- enabled: true,
- createdAt: 1,
- updatedAt: 1,
},
'cloudflare-api-token',
+ { fetch: globalThis.fetch },
),
- /Failed to fetch provider models/,
+ { ok: false, error: { kind: 'invalid_response' } },
);
assert.equal(requestCount, 41);
});
@@ -119,9 +197,11 @@ describe('fetchProviderModels', () => {
next_page_token: 'same-token',
});
});
- await assert.rejects(
- fetchProviderModels(cohereConnection(repeated.url), 'cohere-key'),
- /Failed to fetch provider models/,
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(cohereConnection(repeated.url), 'cohere-key', {
+ fetch: globalThis.fetch,
+ }),
+ { ok: false, error: { kind: 'invalid_response' } },
);
assert.equal(repeatedRequests, 2);
@@ -133,9 +213,11 @@ describe('fetchProviderModels', () => {
})),
});
});
- await assert.rejects(
- fetchProviderModels(cohereConnection(oversized.url), 'cohere-key'),
- /Failed to fetch provider models/,
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(cohereConnection(oversized.url), 'cohere-key', {
+ fetch: globalThis.fetch,
+ }),
+ { ok: false, error: { kind: 'invalid_response' } },
);
});
@@ -175,7 +257,7 @@ describe('fetchProviderModels', () => {
respondWithModels();
});
- const models = await fetchProviderModels(fireworksConnection(server.url), 'fireworks-key');
+ const models = await discoverModels(fireworksConnection(server.url), 'fireworks-key');
assert.equal(maxActiveModelRequests, 4);
assert.equal(models.length, 8);
assert.ok(models.some(({ id }) => id === 'accounts/team-0/models/second'));
@@ -191,9 +273,13 @@ describe('fetchProviderModels', () => {
})),
});
});
- await assert.rejects(
- fetchProviderModels(fireworksConnection(tooManyAccounts.url), 'fireworks-key'),
- /Failed to fetch provider models/,
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(
+ fireworksConnection(tooManyAccounts.url),
+ 'fireworks-key',
+ { fetch: globalThis.fetch },
+ ),
+ { ok: false, error: { kind: 'invalid_response' } },
);
const tooManyEntries = await startJsonServer((request, response) => {
@@ -209,9 +295,13 @@ describe('fetchProviderModels', () => {
})),
});
});
- await assert.rejects(
- fetchProviderModels(fireworksConnection(tooManyEntries.url), 'fireworks-key'),
- /Failed to fetch provider models/,
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(
+ fireworksConnection(tooManyEntries.url),
+ 'fireworks-key',
+ { fetch: globalThis.fetch },
+ ),
+ { ok: false, error: { kind: 'invalid_response' } },
);
let repeatedRequests = 0;
@@ -227,37 +317,37 @@ describe('fetchProviderModels', () => {
nextPageToken: 'same-token',
});
});
- await assert.rejects(
- fetchProviderModels(fireworksConnection(repeatedToken.url), 'fireworks-key'),
- /Failed to fetch provider models/,
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(
+ fireworksConnection(repeatedToken.url),
+ 'fireworks-key',
+ { fetch: globalThis.fetch },
+ ),
+ { ok: false, error: { kind: 'invalid_response' } },
);
assert.equal(repeatedRequests, 2);
});
- test('xAI OAuth preserves discovery HTTP status for auth-failure classification', async () => {
+ test('xAI OAuth classifies discovery authentication failure', async () => {
const server = await startJsonServer((_request, response) => {
respondJson(response, 401, { error: 'invalid_token' });
});
- await assert.rejects(
- fetchProviderModels(
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(
{
- slug: 'xai-oauth',
- name: 'xAI OAuth',
providerType: 'xai-oauth',
baseUrl: `${server.url}/v1`,
defaultModel: 'grok-4.5',
- enabled: true,
- createdAt: 1,
- updatedAt: 1,
},
'expired-xai-oauth-token',
+ { fetch: globalThis.fetch },
),
- (error: unknown) => error instanceof ProviderModelDiscoveryHttpError && error.status === 401,
+ { ok: false, error: { kind: 'auth', statusCode: 401 } },
);
});
- test('provider fetch failures throw generalized errors instead of returning fallback models', async () => {
+ test('provider auth failure exposes only its class and status, without secrets or fallback models', async () => {
const server = await startJsonServer((_request, response) => {
respondJson(response, 401, {
error: 'bad token',
@@ -265,15 +355,12 @@ describe('fetchProviderModels', () => {
});
});
- await assert.rejects(
- () => fetchProviderModels({ ...zaiConnection(), baseUrl: server.url }, 'zai-live-secret'),
- (error) => {
- assert.ok(error instanceof Error);
- assert.equal(error.message, 'Authentication failed');
- assert.equal(error.message.includes('zai-live-secret'), false);
- return true;
- },
+ const outcome = await runConnectionModelDiscoveryEffect(
+ { ...zaiConnection(), baseUrl: server.url },
+ 'zai-live-secret',
+ { fetch: globalThis.fetch },
);
+ assert.deepEqual(outcome, { ok: false, error: { kind: 'auth', statusCode: 401 } });
});
test('Codex OAuth discovers models from the chatgpt.com/backend-api/codex/models endpoint', async () => {
@@ -291,7 +378,7 @@ describe('fetchProviderModels', () => {
});
});
- const models = await fetchProviderModels(
+ const models = await discoverModels(
{
slug: 'openai-codex',
name: 'Codex OAuth',
@@ -325,7 +412,7 @@ describe('fetchProviderModels', () => {
capturedAccountId = request.headers['chatgpt-account-id'];
respondJson(response, 200, { models: [{ slug: 'gpt-5.6-sol' }] });
});
- await fetchProviderModels(
+ await discoverModels(
{
slug: 'openai-codex',
name: 'Codex OAuth',
@@ -341,25 +428,21 @@ describe('fetchProviderModels', () => {
assert.equal(capturedAccountId, 'acct-42');
});
- test('Codex OAuth discovery surfaces the HTTP status on auth failure for caller classification', async () => {
+ test('Codex OAuth classifies discovery authentication failure', async () => {
const server = await startJsonServer((_request, response) => {
respondJson(response, 401, { error: 'unauthorized' });
});
- await assert.rejects(
- fetchProviderModels(
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(
{
- slug: 'openai-codex',
- name: 'Codex OAuth',
providerType: 'openai-codex',
baseUrl: server.url,
defaultModel: 'gpt-5.6-sol',
- enabled: true,
- createdAt: 1,
- updatedAt: 1,
},
'codex-oauth-token',
+ { fetch: globalThis.fetch },
),
- (err: unknown) => (err as { status?: number }).status === 401,
+ { ok: false, error: { kind: 'auth', statusCode: 401 } },
);
});
@@ -409,7 +492,6 @@ describe('fetchProviderModels', () => {
);
assert.deepEqual(outcome, { ok: false, error: { kind: 'invalid_response' } });
- assert.equal(JSON.stringify(outcome).includes(secret), false);
}
});
@@ -429,7 +511,7 @@ describe('fetchProviderModels', () => {
});
});
- const models = await fetchProviderModels(
+ const models = await discoverModels(
{ ...zaiConnection(), baseUrl: server.url },
'zai-live-secret',
);
@@ -469,7 +551,7 @@ describe('fetchProviderModels', () => {
});
});
- const models = await fetchProviderModels(
+ const models = await discoverModels(
{ ...zaiConnection(), baseUrl: server.url },
'zai-live-secret',
);
diff --git a/packages/runtime/src/__tests__/provider-conformance.test.ts b/packages/runtime/src/__tests__/provider-conformance.test.ts
index d7e5e3f542..d29f32033d 100644
--- a/packages/runtime/src/__tests__/provider-conformance.test.ts
+++ b/packages/runtime/src/__tests__/provider-conformance.test.ts
@@ -27,7 +27,8 @@ import { PROVIDER_REGISTRY, type LlmConnection } from '@maka/core/llm-connection
import { anthropic } from '@ai-sdk/anthropic';
import { generateText, isStepCount, streamText, tool, type ModelMessage } from 'ai';
import { z } from 'zod';
-import { fetchProviderModels, runConnectionModelDiscoveryEffect } from '../model-fetcher.js';
+import { runConnectionModelDiscoveryEffect } from '../model-fetcher.js';
+import { discoverModels } from './model-discovery-fixture.js';
import { resetStreamUsageFallbackMemory } from '../stream-usage-fallback-fetch.js';
import { buildProviderOptions, getAIModel } from '../model-factory.js';
import { resolveOAuthSubscriptionAccessToken } from '../subscription-credentials.js';
@@ -990,7 +991,7 @@ describe('models.dev provider conformance', () => {
createdAt: 1,
updatedAt: 1,
};
- connection.models = await fetchProviderModels(connection, 'github-account-token');
+ connection.models = await discoverModels(connection, 'github-account-token');
assert.deepEqual(connection.models.map((model) => model.id).sort(), [
'claude-sonnet-4.6',
'gpt-5.4',
@@ -1346,7 +1347,7 @@ describe('models.dev provider conformance', () => {
updatedAt: 1,
};
- assert.deepEqual(await fetchProviderModels(connection, 'opencode-test-key'), [{ id: modelId }]);
+ assert.deepEqual(await discoverModels(connection, 'opencode-test-key'), [{ id: modelId }]);
const result = await generateText({
model: getAIModel({ connection, apiKey: 'opencode-test-key', modelId }),
prompt: 'Call echo with hello.',
@@ -1842,7 +1843,7 @@ describe('models.dev provider conformance', () => {
updatedAt: 1,
};
- const models = await fetchProviderModels(connection, 'hf-test-token');
+ const models = await discoverModels(connection, 'hf-test-token');
assert.deepEqual(models, [{ id: discoveredModelId, capabilities: { functionCalling: true } }]);
const result = await generateText({
diff --git a/packages/runtime/src/__tests__/provider-contract-matrix.test.ts b/packages/runtime/src/__tests__/provider-contract-matrix.test.ts
index cea0e2eeed..448560a503 100644
--- a/packages/runtime/src/__tests__/provider-contract-matrix.test.ts
+++ b/packages/runtime/src/__tests__/provider-contract-matrix.test.ts
@@ -32,7 +32,7 @@ import {
import { PROVIDER_REGISTRY } from '@maka/core/llm-connections';
import { generateText, isStepCount, tool } from 'ai';
import { z } from 'zod';
-import { fetchProviderModels } from '../model-fetcher.js';
+import { discoverModels } from './model-discovery-fixture.js';
import { getAIModel } from '../model-factory.js';
import {
closeAllJsonServers,
@@ -180,7 +180,7 @@ async function runGeneratedDiscovery(
);
});
const connection = baseConnection(row, server.url);
- const models = await fetchProviderModels(connection, credentialCase.apiKey);
+ const models = await discoverModels(connection, credentialCase.apiKey);
if (handlerErrors.length > 0) throw handlerErrors[0];
assert.ok(requestCount >= 1, `${where} must request the model list`);
assert.deepEqual(
@@ -321,7 +321,7 @@ async function assertFallbackDiscoveryMakesNoRequest(row: ProviderContractRow):
respondJson(response, 500, { error: 'fallback discovery must not reach the network' });
});
const connection = baseConnection(row, server.url);
- const models = await fetchProviderModels(connection, API_KEY);
+ const models = await discoverModels(connection, API_KEY);
assert.equal(
requestCount,
0,
diff --git a/packages/runtime/src/__tests__/provider-contract-overrides.ts b/packages/runtime/src/__tests__/provider-contract-overrides.ts
index 13891c5d69..a8dda7c575 100644
--- a/packages/runtime/src/__tests__/provider-contract-overrides.ts
+++ b/packages/runtime/src/__tests__/provider-contract-overrides.ts
@@ -35,7 +35,7 @@ import assert from 'node:assert/strict';
import type { LlmConnection } from '@maka/core/llm-connections';
import { generateText, isStepCount, streamText, tool } from 'ai';
import { z } from 'zod';
-import { fetchProviderModels } from '../model-fetcher.js';
+import { discoverModels } from './model-discovery-fixture.js';
import { buildProviderOptions, getAIModel } from '../model-factory.js';
import { buildSubscriptionModelFetch } from '../subscription-model-fetch.js';
import {
@@ -349,7 +349,7 @@ async function runCloudflareDiscovery(): Promise<void> {
updatedAt: 1,
};
- assert.deepEqual(await fetchProviderModels(connection, 'cloudflare-test-token'), [
+ assert.deepEqual(await discoverModels(connection, 'cloudflare-test-token'), [
{ id: '@cf/meta/llama-text' },
]);
}
@@ -394,7 +394,7 @@ async function runGitHubCopilotDiscovery(): Promise<void> {
});
});
- const models = await fetchProviderModels(
+ const models = await discoverModels(
{
slug: 'github-copilot',
name: 'GitHub Copilot',
@@ -540,7 +540,7 @@ async function runGitHubCopilotWire(): Promise<void> {
createdAt: 1,
updatedAt: 1,
};
- const models = await fetchProviderModels(connection, 'github-account-token');
+ const models = await discoverModels(connection, 'github-account-token');
connection.models = models;
const modelFetch = buildSubscriptionModelFetch({
connection,
@@ -749,7 +749,7 @@ async function runFireworksDiscovery(): Promise<void> {
updatedAt: 1,
};
- const models = await fetchProviderModels(connection, 'fireworks-test-key');
+ const models = await discoverModels(connection, 'fireworks-test-key');
assert.deepEqual(models, [
{
id: 'accounts/acme/models/custom-agent',
@@ -881,7 +881,7 @@ async function assertOllamaModelContract(
};
assert.deepEqual(
- await fetchProviderModels(connection, ''),
+ await discoverModels(connection, ''),
discoveredModelIds.map((id) => ({ id })),
);
@@ -1009,7 +1009,7 @@ async function runCohereDiscovery(): Promise<void> {
updatedAt: 1,
};
- const models = await fetchProviderModels(connection, 'cohere-test-key');
+ const models = await discoverModels(connection, 'cohere-test-key');
assert.deepEqual(models, [
{ id: modelId, contextWindow: 128_000 },
{ id: 'command-a-reasoning-08-2025', contextWindow: 256_000 },
diff --git a/packages/runtime/src/model-fetcher.ts b/packages/runtime/src/model-fetcher.ts
index 03213ca944..e912d2aa9f 100644
--- a/packages/runtime/src/model-fetcher.ts
+++ b/packages/runtime/src/model-fetcher.ts
@@ -23,15 +23,13 @@ import {
effectiveBaseUrl,
isModelModality,
providerAuthSupportsApiKey,
- type LlmConnection,
type ModelInfo,
type ModelModality,
} from '@maka/core/llm-connections';
-import { generalizedErrorMessage } from '@maka/core/redaction';
import {
CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION,
CONNECTION_MODEL_ID_MAX_LENGTH,
- normalizeConnectionModelDiscoveryResult,
+ decodeConnectionModels,
} from '@maka/core/runtime-policy';
import { anthropicV1Url, googleApiUrl } from './provider-urls.js';
import { openAiCodexHeaders } from './subscription-auth.js';
@@ -44,7 +42,6 @@ import {
fetchForConnectionEffect,
type ConnectionEffectFetch,
type ConnectionEffectFetchDependency,
- type ConnectionEffectFetchOptions,
type ConnectionEffectResponse,
} from './connection-effect-fetch.js';
import {
@@ -138,28 +135,6 @@ type FireworksModelDiscovery = Extract<
{ kind: 'fireworks' }
>;
-export async function fetchProviderModels(
- connection: LlmConnection,
- apiKey: string,
- options: ConnectionEffectFetchOptions = {},
-): Promise<ModelInfo[]> {
- try {
- return normalizeDiscoveredModels(
- await fetchProviderModelsStrict(connection, apiKey, options.fetch),
- );
- } catch (error) {
- // Preserve status-bearing discovery errors so the sync layer can classify
- // auth/protocol/network failures; only wrap unknown errors for display.
- if (
- error instanceof OpenAiCodexDiscoveryError ||
- error instanceof ProviderModelDiscoveryHttpError
- ) {
- throw error;
- }
- throw new Error(generalizedErrorMessage(error, 'Failed to fetch provider models'));
- }
-}
-
export async function runConnectionModelDiscoveryEffect(
connection: ConnectionEffectConnection,
apiKey: string,
@@ -262,9 +237,6 @@ async function fetchProviderModelsStrict(
);
if (!r.ok) {
await r.cancel();
- if (connection.providerType === 'xai-oauth') {
- throw new ProviderModelDiscoveryHttpError(r.status);
- }
throw new ConnectionEffectHttpError(r.status);
}
const data = await readProviderJson<{ data?: unknown } | unknown[]>(r);
@@ -372,7 +344,7 @@ function cloudflareModelsUrl(baseUrl: string, page: number): string {
return url.toString();
}
-function normalizeDiscoveredModels(models: ModelInfo[]): ModelInfo[] {
+function normalizeConnectionEffectModels(models: ModelInfo[]): readonly ModelInfo[] {
const unique = new Map<string, ModelInfo>();
for (const model of models) {
if (typeof model?.id !== 'string') continue;
@@ -390,16 +362,8 @@ function normalizeDiscoveredModels(models: ModelInfo[]): ModelInfo[] {
throw new ConnectionEffectInvalidResponseError('Provider returned too many models');
}
}
- return [...unique.values()];
-}
-
-function normalizeConnectionEffectModels(models: ModelInfo[]): readonly ModelInfo[] {
try {
- return normalizeConnectionModelDiscoveryResult({
- models: normalizeDiscoveredModels(models),
- source: 'fetched',
- fetchedAt: 0,
- }).models;
+ return decodeConnectionModels([...unique.values()]);
} catch (error) {
throw new ConnectionEffectInvalidResponseError('Provider returned invalid model metadata', {
cause: error,
@@ -452,26 +416,6 @@ type RawOpenAiCodexModel = {
context_window?: unknown;
};
-/**
- * Discovery error carrying the HTTP status, so callers (syncOpenAiCodexConnection)
- * can classify auth failures (401/403) vs protocol errors (4xx) vs transient
- * network failures without string-matching the message.
- */
-export class OpenAiCodexDiscoveryError extends ConnectionEffectHttpError {
- constructor(status: number) {
- super(status);
- this.name = 'OpenAiCodexDiscoveryError';
- }
-}
-
-/** Structured status for standard provider `/models` endpoints. */
-export class ProviderModelDiscoveryHttpError extends ConnectionEffectHttpError {
- constructor(status: number) {
- super(status);
- this.name = 'ProviderModelDiscoveryHttpError';
- }
-}
-
/**
* Discover models from the ChatGPT/Codex OAuth backend
* (`chatgpt.com/backend-api/codex/models`). Unlike the public OpenAI API
@@ -500,7 +444,7 @@ export async function fetchOpenAiCodexModels(
);
if (!response.ok) {
await response.cancel();
- throw new OpenAiCodexDiscoveryError(response.status);
+ throw new ConnectionEffectHttpError(response.status);
}
const payload = await readProviderJson<{ models?: unknown }>(response);
const models = providerObjectArray<RawOpenAiCodexModel>(
@@ -517,7 +461,7 @@ export async function fetchOpenAiCodexModels(
visible.sort((a, b) => priorityOfOpenAiCodexModel(a) - priorityOfOpenAiCodexModel(b));
return visible.map((model) => {
const entry: ModelInfo = { id: (model.slug as string).trim() };
- const contextWindow = contextWindowOfOpenAiCodexModel(model);
+ const contextWindow = providerTokenLimit(model.context_window);
if (contextWindow !== undefined) entry.contextWindow = contextWindow;
return entry;
});
@@ -529,12 +473,9 @@ function priorityOfOpenAiCodexModel(model: RawOpenAiCodexModel): number {
: 10_000;
}
-function contextWindowOfOpenAiCodexModel(model: RawOpenAiCodexModel): number | undefined {
- return typeof model.context_window === 'number' &&
- Number.isFinite(model.context_window) &&
- model.context_window > 0
- ? model.context_window
- : undefined;
+function providerTokenLimit(value: unknown): number | undefined {
+ // Invalid upstream metadata makes no claim about the model's limit.
+ return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : undefined;
}
function toGitHubCopilotModelInfo(model: RawGitHubCopilotModel): ModelInfo[] {
@@ -581,15 +522,16 @@ function toGitHubCopilotModelInfo(model: RawGitHubCopilotModel): ModelInfo[] {
limits?.vision?.supported_media_types?.some(
(type) => typeof type === 'string' && type.startsWith('image/'),
) === true;
- const contextWindow = limits?.max_context_window_tokens ?? limits?.max_prompt_tokens;
+ const contextWindow = providerTokenLimit(
+ limits?.max_context_window_tokens ?? limits?.max_prompt_tokens,
+ );
+ const maxOutputTokens = providerTokenLimit(limits?.max_output_tokens);
return [
{
id: model.id,
...(model.name ? { displayName: model.name } : {}),
- ...(typeof contextWindow === 'number' ? { contextWindow } : {}),
- ...(typeof limits?.max_output_tokens === 'number'
- ? { maxOutputTokens: limits.max_output_tokens }
- : {}),
+ ...(contextWindow === undefined ? {} : { contextWindow }),
+ ...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
apiProtocol,
capabilities: { vision, reasoning, functionCalling: true },
},
@@ -670,12 +612,11 @@ async function fetchCohereModels(
}
assertOptionalArray(model.endpoints, 'model endpoints');
if (!model.endpoints?.includes('chat')) return [];
+ const contextWindow = providerTokenLimit(model.context_length);
return [
{
id: model.name,
- ...(typeof model.context_length === 'number'
- ? { contextWindow: model.context_length }
- : {}),
+ ...(contextWindow === undefined ? {} : { contextWindow }),
},
];
}),
@@ -825,6 +766,7 @@ async function fetchFireworksModels(
return modelLists.flat().flatMap((model) => {
if (!model.name) return [];
+ const contextWindow = providerTokenLimit(model.contextLength);
const capabilities: NonNullable<ModelInfo['capabilities']> = {};
if (typeof model.supportsImageInput === 'boolean')
capabilities.vision = model.supportsImageInput;
@@ -834,7 +776,7 @@ async function fetchFireworksModels(
{
id: model.name,
...(model.displayName ? { displayName: model.displayName } : {}),
- ...(typeof model.contextLength === 'number' ? { contextWindow: model.contextLength } : {}),
+ ...(contextWindow === undefined ? {} : { contextWindow }),
...(Object.keys(capabilities).length ? { capabilities } : {}),
},
];
@@ -850,7 +792,8 @@ function toModelInfo(model: RawProviderModel): ModelInfo | null {
model.providers,
'model providers',
);
- const contextWindow = model.context_length ?? model.context_window;
+ const contextWindow = providerTokenLimit(model.context_length ?? model.context_window);
+ const maxOutputTokens = providerTokenLimit(model.max_tokens);
const capabilities: NonNullable<ModelInfo['capabilities']> = {};
if (model.input_modalities?.includes('image')) capabilities.vision = true;
if (typeof model.capabilities?.reasoning === 'boolean')
@@ -885,8 +828,8 @@ function toModelInfo(model: RawProviderModel): ModelInfo | null {
return {
id: model.id,
...(model.display_name || model.name ? { displayName: model.display_name ?? model.name } : {}),
- ...(typeof contextWindow === 'number' ? { contextWindow } : {}),
- ...(typeof model.max_tokens === 'number' ? { maxOutputTokens: model.max_tokens } : {}),
+ ...(contextWindow === undefined ? {} : { contextWindow }),
+ ...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
...(Object.keys(capabilities).length ? { capabilities } : {}),
};
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org