You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maka.apache.org by as...@apache.org on 2026/09/16 07:05:34 UTC
(maka) branch main updated: fix(runtime): unify model discovery catalog validation (#5376)
This is an automated email from the ASF dual-hosted git repository.
Astro-Han pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/maka.git
The following commit(s) were added to refs/heads/main by this push:
new 4410c3a2d1 fix(runtime): unify model discovery catalog validation (#5376)
4410c3a2d1 is described below
commit 4410c3a2d19d20cfdc6b7815bdd2d72d33a7460f
Author: AstroHan <le...@outlook.com>
AuthorDate: Wed Sep 16 15:05:29 2026 +0800
fix(runtime): unify model discovery catalog validation (#5376)
Unify strict model catalog validation and provider token-limit adaptation, preserve valid fallback limits, and remove redundant discovery paths and tests.
Generated-by: Codex
---
.../src/__tests__/runtime-policy-codec.test.ts | 4 +-
packages/core/src/runtime-policy.ts | 1 +
.../src/runtime-policy/connection-catalog-codec.ts | 33 +--
.../connection-effect-coordinator.test.ts | 73 +++++
.../src/__tests__/model-discovery-fixture.ts | 33 +++
.../runtime/src/__tests__/model-fetcher.test.ts | 330 +++++++++++----------
.../src/__tests__/provider-conformance.test.ts | 9 +-
.../src/__tests__/provider-contract-matrix.test.ts | 6 +-
.../src/__tests__/provider-contract-overrides.ts | 14 +-
packages/runtime/src/model-fetcher.ts | 134 +++------
10 files changed, 342 insertions(+), 295 deletions(-)
diff --git a/packages/core/src/__tests__/runtime-policy-codec.test.ts b/packages/core/src/__tests__/runtime-policy-codec.test.ts
index 7efd198ba1..2fd2e91173 100644
--- a/packages/core/src/__tests__/runtime-policy-codec.test.ts
+++ b/packages/core/src/__tests__/runtime-policy-codec.test.ts
@@ -506,8 +506,10 @@ test('normalizes exact bounded model discovery results', () => {
},
);
for (const invalid of [
+ { models: [{ id: 'duplicate' }, { id: 'duplicate' }], source: 'fetched', fetchedAt: 42 },
+ { models: [{ id: 'invalid', contextWindow: 0 }], source: 'fetched', fetchedAt: 42 },
{
- models: [{ id: 'duplicate' }, { id: 'duplicate' }],
+ models: Array.from({ length: 2049 }, (_, i) => ({ id: `model-${i}` })),
source: 'fetched',
fetchedAt: 42,
},
diff --git a/packages/core/src/runtime-policy.ts b/packages/core/src/runtime-policy.ts
index dc4491bafb..72c6a1e51c 100644
--- a/packages/core/src/runtime-policy.ts
+++ b/packages/core/src/runtime-policy.ts
@@ -70,6 +70,7 @@ export {
decodeConnectionCredentialTarget,
decodeModelOverridesTable,
decodeConnectionModel,
+ decodeConnectionModels,
decodeConnectionName,
decodeConnectionSlug,
decodeConnectionTarget,
diff --git a/packages/core/src/runtime-policy/connection-catalog-codec.ts b/packages/core/src/runtime-policy/connection-catalog-codec.ts
index c8e2ddce86..813dce30d2 100644
--- a/packages/core/src/runtime-policy/connection-catalog-codec.ts
+++ b/packages/core/src/runtime-policy/connection-catalog-codec.ts
@@ -431,16 +431,7 @@ export function decodeCanonicalConnectionCatalogEntry(value: unknown): Connectio
? {}
: { requestBodyOverlay: item.requestBodyOverlay }),
});
- if (
- !Array.isArray(item.models) ||
- item.models.length > CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION
- ) {
- throw domainError('connection models must be a bounded array');
- }
- const models = item.models.map(decodeConnectionModel);
- if (new Set(models.map((model) => model.id)).size !== models.length) {
- throw domainError('connection model ids must be unique');
- }
+ const models = decodeConnectionModels(item.models);
if (
item.modelSource !== undefined &&
item.modelSource !== 'fetched' &&
@@ -722,20 +713,22 @@ export function decodeConnectionTestSummary(value: unknown): ConnectionTestSumma
};
}
+export function decodeConnectionModels(value: unknown): ConnectionModel[] {
+ if (!Array.isArray(value) || value.length > CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION) {
+ throw domainError('connection models must be a bounded array');
+ }
+ const models = value.map(decodeConnectionModel);
+ if (new Set(models.map((model) => model.id)).size !== models.length) {
+ throw domainError('connection model ids must be unique');
+ }
+ return models;
+}
+
export function normalizeConnectionModelDiscoveryResult(
value: unknown,
): ConnectionModelDiscoveryResult {
const item = exactRecord(value, 'model discovery result', ['models', 'source', 'fetchedAt']);
- if (
- !Array.isArray(item.models) ||
- item.models.length > CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION
- ) {
- throw domainError('model discovery models must be a bounded array');
- }
- const models = item.models.map(decodeConnectionModel);
- if (new Set(models.map((model) => model.id)).size !== models.length) {
- throw domainError('model discovery model ids must be unique');
- }
+ const models = decodeConnectionModels(item.models);
if (item.source !== 'fetched' && item.source !== 'fallback') {
throw domainError('model discovery source is invalid');
}
diff --git a/packages/runtime-host/src/__tests__/connection-effect-coordinator.test.ts b/packages/runtime-host/src/__tests__/connection-effect-coordinator.test.ts
index 420b2d964c..09c877a3f7 100644
--- a/packages/runtime-host/src/__tests__/connection-effect-coordinator.test.ts
+++ b/packages/runtime-host/src/__tests__/connection-effect-coordinator.test.ts
@@ -51,6 +51,79 @@ const context: ConnectionContext = {
acquireResidency: () => ({ release: () => undefined }),
};
+test('Fireworks onboarding keeps an unrelated model with unknown limits and persists the selected K3', async () => {
+ const base = await mkdtemp(join(tmpdir(), 'maka-fireworks-onboarding-'));
+ const capability = await resolveStorageRoot({
+ path: join(base, 'interactive'),
+ kind: 'interactive',
+ });
+ const k3 = 'accounts/fireworks/models/kimi-k3';
+ const other = 'accounts/fireworks/models/qwen3p8-max';
+ const requests: string[] = [];
+ try {
+ for (const reopen of [false, true]) {
+ const owner = await tryAcquireInteractiveRootOwner(capability);
+ assert.ok(owner);
+ try {
+ const stores = await openInteractiveRuntimePolicyStoresForWrite(owner.lease);
+ if (!reopen) {
+ const coordinator = new HostConnectionEffectCoordinator({
+ stores,
+ activation: new RuntimePolicyActivationGate(),
+ oauthCredentials: new HostOAuthExecutionAuthority(stores),
+ createTransport: () => ({
+ fetch: async (input) => {
+ const path = new URL(String(input)).pathname;
+ requests.push(path);
+ if (path === '/v1/accounts') return Response.json({ accounts: [] });
+ assert.equal(path, '/v1/accounts/fireworks/models');
+ return Response.json({
+ models: [
+ { name: k3, contextLength: 1048576 },
+ { name: other, contextLength: 0 },
+ ],
+ });
+ },
+ close: async () => undefined,
+ }),
+ });
+ try {
+ const saved = await coordinator.handlers['connection.onboarding.save'](
+ {
+ target: {
+ kind: 'create',
+ providerType: 'fireworks-ai',
+ slug: 'fireworks',
+ name: 'Fireworks',
+ },
+ apiKey: 'fixture-key',
+ baseUrl: null,
+ enabledModelIds: [k3],
+ },
+ context,
+ );
+ assertSaved(saved);
+ } finally {
+ await coordinator.close();
+ }
+ }
+ const catalog = await stores.connectionCatalog.getSnapshot();
+ assert.equal(catalog.connections.length, 1);
+ assert.deepEqual(catalog.connections[0]?.enabledModelIds, [k3]);
+ assert.deepEqual(catalog.connections[0]?.models, [
+ { id: k3, contextWindow: 1048576 },
+ { id: other },
+ ]);
+ } finally {
+ await owner.close();
+ }
+ }
+ assert.deepEqual(requests, ['/v1/accounts', '/v1/accounts/fireworks/models']);
+ } finally {
+ await rm(base, { recursive: true, force: true });
+ }
+});
+
test('verifies a first-run API key without persisting a connection or credential', async () => {
await withFixture(async ({ stores }) => {
let observed: { slug: string; secret: string } | undefined;
diff --git a/packages/runtime/src/__tests__/model-discovery-fixture.ts b/packages/runtime/src/__tests__/model-discovery-fixture.ts
new file mode 100644
index 0000000000..fc09702926
--- /dev/null
+++ b/packages/runtime/src/__tests__/model-discovery-fixture.ts
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import assert from 'node:assert/strict';
+import type { LlmConnection, ModelInfo } from '@maka/core/llm-connections';
+import { runConnectionModelDiscoveryEffect } from '../model-fetcher.js';
+
+export async function discoverModels(
+ connection: LlmConnection,
+ apiKey: string,
+): Promise<ModelInfo[]> {
+ const outcome = await runConnectionModelDiscoveryEffect(connection, apiKey, {
+ fetch: globalThis.fetch,
+ });
+ if (!outcome.ok) assert.fail(outcome.error.kind);
+ return [...outcome.models];
+}
diff --git a/packages/runtime/src/__tests__/model-fetcher.test.ts b/packages/runtime/src/__tests__/model-fetcher.test.ts
index 6e615175a6..963f071acd 100644
--- a/packages/runtime/src/__tests__/model-fetcher.test.ts
+++ b/packages/runtime/src/__tests__/model-fetcher.test.ts
@@ -18,22 +18,122 @@
*/
import assert from 'node:assert/strict';
-import { createServer, type IncomingMessage, type ServerResponse } from 'node:http';
import { after, describe, test } from 'node:test';
import type { LlmConnection } from '@maka/core/llm-connections';
-import {
- fetchProviderModels,
- ProviderModelDiscoveryHttpError,
- runConnectionModelDiscoveryEffect,
-} from '../model-fetcher.js';
-
-const servers: Array<{ close(): Promise<void> }> = [];
-
-after(async () => {
- await Promise.all(servers.map((server) => server.close()));
-});
+import { runConnectionModelDiscoveryEffect } from '../model-fetcher.js';
+import { discoverModels } from './model-discovery-fixture.js';
+
+import { startJsonServer, respondJson, closeAllJsonServers } from './conformance-harness.js';
+
+after(closeAllJsonServers);
+
+describe('model discovery', () => {
+ test('all token metadata adapters preserve valid limits and omit invalid optional limits', async () => {
+ const values = [
+ [undefined, undefined],
+ [null, undefined],
+ ['128000', undefined],
+ [0, undefined],
+ [-1, undefined],
+ [1.5, undefined],
+ [Number.MAX_SAFE_INTEGER + 1, undefined],
+ [1, 1],
+ [128000, 128000],
+ [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER],
+ ] as const;
+ const cases = [
+ {
+ providerType: 'fireworks-ai',
+ row: (id: string, value: unknown) => ({ name: id, contextLength: value }),
+ key: 'models',
+ output: false,
+ },
+ {
+ providerType: 'cohere',
+ row: (id: string, value: unknown) => ({
+ name: id,
+ context_length: value,
+ endpoints: ['chat'],
+ }),
+ key: 'models',
+ output: false,
+ },
+ {
+ providerType: 'openai-compatible',
+ row: (id: string, value: unknown, fallback?: number) => ({
+ id,
+ context_length: value,
+ context_window: fallback,
+ max_tokens: value,
+ }),
+ key: 'data',
+ output: true,
+ },
+ {
+ providerType: 'github-copilot',
+ row: (id: string, value: unknown, fallback?: number) => ({
+ id,
+ model_picker_enabled: true,
+ supported_endpoints: ['/responses'],
+ capabilities: {
+ supports: { tool_calls: true },
+ limits: {
+ max_context_window_tokens: value,
+ max_prompt_tokens: fallback,
+ max_output_tokens: value,
+ },
+ },
+ }),
+ key: 'data',
+ output: true,
+ },
+ {
+ providerType: 'openai-codex',
+ row: (id: string, value: unknown) => ({ slug: id, context_window: value }),
+ key: 'models',
+ output: false,
+ },
+ ] as const;
+ for (const fixture of cases) {
+ const fallbacks =
+ fixture.providerType === 'openai-compatible' || fixture.providerType === 'github-copilot'
+ ? [undefined, 32768]
+ : [undefined];
+ for (const fallback of fallbacks) {
+ const outcome = await runConnectionModelDiscoveryEffect(
+ { providerType: fixture.providerType, baseUrl: 'https://fixture.invalid/v1' },
+ 'fixture-key',
+ {
+ fetch: async (input) =>
+ Response.json(
+ new URL(String(input)).pathname === '/v1/accounts'
+ ? { accounts: [] }
+ : {
+ [fixture.key]: values.map(([value], i) =>
+ fixture.row(`model-${i}`, value, fallback),
+ ),
+ },
+ ),
+ },
+ );
+ assert.ok(outcome.ok, fixture.providerType);
+ assert.deepEqual(
+ outcome.models.map(({ id, contextWindow, maxOutputTokens }) => ({
+ id,
+ contextWindow,
+ maxOutputTokens,
+ })),
+ values.map(([, expected], i) => ({
+ id: `model-${i}`,
+ contextWindow: expected ?? fallback,
+ maxOutputTokens: fixture.output ? expected : undefined,
+ })),
+ fixture.providerType,
+ );
+ }
+ }
+ });
-describe('fetchProviderModels', () => {
test('Cloudflare Workers AI accepts exactly 2,048 models and rejects the first excess item', async () => {
for (const modelCount of [2_048, 2_049]) {
let requestCount = 0;
@@ -50,66 +150,27 @@ describe('fetchProviderModels', () => {
respondJson(response, 200, { success: true, result });
});
- const request = fetchProviderModels(
+ const request = runConnectionModelDiscoveryEffect(
{
- slug: 'cloudflare-workers-ai',
- name: 'Cloudflare Workers AI',
providerType: 'cloudflare-workers-ai',
baseUrl: `${server.url}/client/v4/accounts/account-123/ai/v1`,
defaultModel: '@cf/example/default',
- enabled: true,
- createdAt: 1,
- updatedAt: 1,
},
'cloudflare-api-token',
+ { fetch: globalThis.fetch },
);
if (modelCount === 2_048) {
- assert.equal((await request).length, 2_048);
+ const outcome = await request;
+ assert.ok(outcome.ok);
+ assert.equal(outcome.models.length, 2_048);
assert.equal(requestCount, 42);
} else {
- await assert.rejects(request, /Failed to fetch provider models/);
+ assert.deepEqual(await request, { ok: false, error: { kind: 'invalid_response' } });
assert.equal(requestCount, 41);
}
}
});
- test('Cloudflare Workers AI bounds pagination before an oversized catalog can be persisted', async () => {
- let requestCount = 0;
- const server = await startJsonServer((_request, response) => {
- requestCount += 1;
- respondJson(response, 200, {
- success: true,
- result: Array.from({ length: 50 }, (_, index) => ({
- name: `@cf/example/page-${requestCount}-model-${index}`,
- })),
- result_info: {
- page: requestCount,
- per_page: 50,
- count: 50,
- total_count: 10_000,
- },
- });
- });
-
- await assert.rejects(
- fetchProviderModels(
- {
- slug: 'cloudflare-workers-ai',
- name: 'Cloudflare Workers AI',
- providerType: 'cloudflare-workers-ai',
- baseUrl: `${server.url}/client/v4/accounts/account-123/ai/v1`,
- defaultModel: '@cf/example/default',
- enabled: true,
- createdAt: 1,
- updatedAt: 1,
- },
- 'cloudflare-api-token',
- ),
- /Failed to fetch provider models/,
- );
- assert.equal(requestCount, 41);
- });
-
test('Cohere rejects repeated page tokens or excess entries', async () => {
let repeatedRequests = 0;
const repeated = await startJsonServer((_request, response) => {
@@ -119,9 +180,11 @@ describe('fetchProviderModels', () => {
next_page_token: 'same-token',
});
});
- await assert.rejects(
- fetchProviderModels(cohereConnection(repeated.url), 'cohere-key'),
- /Failed to fetch provider models/,
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(cohereConnection(repeated.url), 'cohere-key', {
+ fetch: globalThis.fetch,
+ }),
+ { ok: false, error: { kind: 'invalid_response' } },
);
assert.equal(repeatedRequests, 2);
@@ -133,9 +196,11 @@ describe('fetchProviderModels', () => {
})),
});
});
- await assert.rejects(
- fetchProviderModels(cohereConnection(oversized.url), 'cohere-key'),
- /Failed to fetch provider models/,
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(cohereConnection(oversized.url), 'cohere-key', {
+ fetch: globalThis.fetch,
+ }),
+ { ok: false, error: { kind: 'invalid_response' } },
);
});
@@ -175,7 +240,7 @@ describe('fetchProviderModels', () => {
respondWithModels();
});
- const models = await fetchProviderModels(fireworksConnection(server.url), 'fireworks-key');
+ const models = await discoverModels(fireworksConnection(server.url), 'fireworks-key');
assert.equal(maxActiveModelRequests, 4);
assert.equal(models.length, 8);
assert.ok(models.some(({ id }) => id === 'accounts/team-0/models/second'));
@@ -191,9 +256,13 @@ describe('fetchProviderModels', () => {
})),
});
});
- await assert.rejects(
- fetchProviderModels(fireworksConnection(tooManyAccounts.url), 'fireworks-key'),
- /Failed to fetch provider models/,
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(
+ fireworksConnection(tooManyAccounts.url),
+ 'fireworks-key',
+ { fetch: globalThis.fetch },
+ ),
+ { ok: false, error: { kind: 'invalid_response' } },
);
const tooManyEntries = await startJsonServer((request, response) => {
@@ -209,9 +278,13 @@ describe('fetchProviderModels', () => {
})),
});
});
- await assert.rejects(
- fetchProviderModels(fireworksConnection(tooManyEntries.url), 'fireworks-key'),
- /Failed to fetch provider models/,
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(
+ fireworksConnection(tooManyEntries.url),
+ 'fireworks-key',
+ { fetch: globalThis.fetch },
+ ),
+ { ok: false, error: { kind: 'invalid_response' } },
);
let repeatedRequests = 0;
@@ -227,53 +300,36 @@ describe('fetchProviderModels', () => {
nextPageToken: 'same-token',
});
});
- await assert.rejects(
- fetchProviderModels(fireworksConnection(repeatedToken.url), 'fireworks-key'),
- /Failed to fetch provider models/,
- );
- assert.equal(repeatedRequests, 2);
- });
-
- test('xAI OAuth preserves discovery HTTP status for auth-failure classification', async () => {
- const server = await startJsonServer((_request, response) => {
- respondJson(response, 401, { error: 'invalid_token' });
- });
-
- await assert.rejects(
- fetchProviderModels(
- {
- slug: 'xai-oauth',
- name: 'xAI OAuth',
- providerType: 'xai-oauth',
- baseUrl: `${server.url}/v1`,
- defaultModel: 'grok-4.5',
- enabled: true,
- createdAt: 1,
- updatedAt: 1,
- },
- 'expired-xai-oauth-token',
+ assert.deepEqual(
+ await runConnectionModelDiscoveryEffect(
+ fireworksConnection(repeatedToken.url),
+ 'fireworks-key',
+ { fetch: globalThis.fetch },
),
- (error: unknown) => error instanceof ProviderModelDiscoveryHttpError && error.status === 401,
+ { ok: false, error: { kind: 'invalid_response' } },
);
+ assert.equal(repeatedRequests, 2);
});
- test('provider fetch failures throw generalized errors instead of returning fallback models', async () => {
+ test('discovery classifies HTTP authentication failure without secrets or fallback models', async () => {
const server = await startJsonServer((_request, response) => {
respondJson(response, 401, {
- error: 'bad token',
- authorization: 'Bearer zai-live-secret',
+ authorization: 'Bearer fixture-secret',
+ error: 'invalid_token',
});
});
-
- await assert.rejects(
- () => fetchProviderModels({ ...zaiConnection(), baseUrl: server.url }, 'zai-live-secret'),
- (error) => {
- assert.ok(error instanceof Error);
- assert.equal(error.message, 'Authentication failed');
- assert.equal(error.message.includes('zai-live-secret'), false);
- return true;
- },
- );
+ for (const providerType of ['xai-oauth', 'openai-codex', 'zai-coding-plan'] as const) {
+ const outcome = await runConnectionModelDiscoveryEffect(
+ { providerType, baseUrl: server.url },
+ 'fixture-secret',
+ { fetch: globalThis.fetch },
+ );
+ assert.deepEqual(
+ outcome,
+ { ok: false, error: { kind: 'auth', statusCode: 401 } },
+ providerType,
+ );
+ }
});
test('Codex OAuth discovers models from the chatgpt.com/backend-api/codex/models endpoint', async () => {
@@ -291,7 +347,7 @@ describe('fetchProviderModels', () => {
});
});
- const models = await fetchProviderModels(
+ const models = await discoverModels(
{
slug: 'openai-codex',
name: 'Codex OAuth',
@@ -325,7 +381,7 @@ describe('fetchProviderModels', () => {
capturedAccountId = request.headers['chatgpt-account-id'];
respondJson(response, 200, { models: [{ slug: 'gpt-5.6-sol' }] });
});
- await fetchProviderModels(
+ await discoverModels(
{
slug: 'openai-codex',
name: 'Codex OAuth',
@@ -341,28 +397,6 @@ describe('fetchProviderModels', () => {
assert.equal(capturedAccountId, 'acct-42');
});
- test('Codex OAuth discovery surfaces the HTTP status on auth failure for caller classification', async () => {
- const server = await startJsonServer((_request, response) => {
- respondJson(response, 401, { error: 'unauthorized' });
- });
- await assert.rejects(
- fetchProviderModels(
- {
- slug: 'openai-codex',
- name: 'Codex OAuth',
- providerType: 'openai-codex',
- baseUrl: server.url,
- defaultModel: 'gpt-5.6-sol',
- enabled: true,
- createdAt: 1,
- updatedAt: 1,
- },
- 'codex-oauth-token',
- ),
- (err: unknown) => (err as { status?: number }).status === 401,
- );
- });
-
test('connection discovery classifies a wholly policy-blocked Copilot catalog as auth', async () => {
const server = await startJsonServer((_request, response) => {
respondJson(response, 200, {
@@ -409,7 +443,6 @@ describe('fetchProviderModels', () => {
);
assert.deepEqual(outcome, { ok: false, error: { kind: 'invalid_response' } });
- assert.equal(JSON.stringify(outcome).includes(secret), false);
}
});
@@ -429,7 +462,7 @@ describe('fetchProviderModels', () => {
});
});
- const models = await fetchProviderModels(
+ const models = await discoverModels(
{ ...zaiConnection(), baseUrl: server.url },
'zai-live-secret',
);
@@ -469,7 +502,7 @@ describe('fetchProviderModels', () => {
});
});
- const models = await fetchProviderModels(
+ const models = await discoverModels(
{ ...zaiConnection(), baseUrl: server.url },
'zai-live-secret',
);
@@ -483,29 +516,6 @@ describe('fetchProviderModels', () => {
});
});
-async function startJsonServer(
- handler: (request: IncomingMessage, response: ServerResponse) => void,
-): Promise<{ url: string; close(): Promise<void> }> {
- const server = createServer(handler);
- await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
- const address = server.address();
- assert.ok(address && typeof address === 'object');
- const control = {
- url: `http://127.0.0.1:${address.port}`,
- close: () =>
- new Promise<void>((resolve, reject) => {
- server.close((error) => (error ? reject(error) : resolve()));
- }),
- };
- servers.push(control);
- return control;
-}
-
-function respondJson(response: ServerResponse, status: number, body: unknown): void {
- response.writeHead(status, { 'content-type': 'application/json' });
- response.end(JSON.stringify(body));
-}
-
function zaiConnection(): LlmConnection {
return {
slug: 'zai',
diff --git a/packages/runtime/src/__tests__/provider-conformance.test.ts b/packages/runtime/src/__tests__/provider-conformance.test.ts
index d7e5e3f542..d29f32033d 100644
--- a/packages/runtime/src/__tests__/provider-conformance.test.ts
+++ b/packages/runtime/src/__tests__/provider-conformance.test.ts
@@ -27,7 +27,8 @@ import { PROVIDER_REGISTRY, type LlmConnection } from '@maka/core/llm-connection
import { anthropic } from '@ai-sdk/anthropic';
import { generateText, isStepCount, streamText, tool, type ModelMessage } from 'ai';
import { z } from 'zod';
-import { fetchProviderModels, runConnectionModelDiscoveryEffect } from '../model-fetcher.js';
+import { runConnectionModelDiscoveryEffect } from '../model-fetcher.js';
+import { discoverModels } from './model-discovery-fixture.js';
import { resetStreamUsageFallbackMemory } from '../stream-usage-fallback-fetch.js';
import { buildProviderOptions, getAIModel } from '../model-factory.js';
import { resolveOAuthSubscriptionAccessToken } from '../subscription-credentials.js';
@@ -990,7 +991,7 @@ describe('models.dev provider conformance', () => {
createdAt: 1,
updatedAt: 1,
};
- connection.models = await fetchProviderModels(connection, 'github-account-token');
+ connection.models = await discoverModels(connection, 'github-account-token');
assert.deepEqual(connection.models.map((model) => model.id).sort(), [
'claude-sonnet-4.6',
'gpt-5.4',
@@ -1346,7 +1347,7 @@ describe('models.dev provider conformance', () => {
updatedAt: 1,
};
- assert.deepEqual(await fetchProviderModels(connection, 'opencode-test-key'), [{ id: modelId }]);
+ assert.deepEqual(await discoverModels(connection, 'opencode-test-key'), [{ id: modelId }]);
const result = await generateText({
model: getAIModel({ connection, apiKey: 'opencode-test-key', modelId }),
prompt: 'Call echo with hello.',
@@ -1842,7 +1843,7 @@ describe('models.dev provider conformance', () => {
updatedAt: 1,
};
- const models = await fetchProviderModels(connection, 'hf-test-token');
+ const models = await discoverModels(connection, 'hf-test-token');
assert.deepEqual(models, [{ id: discoveredModelId, capabilities: { functionCalling: true } }]);
const result = await generateText({
diff --git a/packages/runtime/src/__tests__/provider-contract-matrix.test.ts b/packages/runtime/src/__tests__/provider-contract-matrix.test.ts
index cea0e2eeed..448560a503 100644
--- a/packages/runtime/src/__tests__/provider-contract-matrix.test.ts
+++ b/packages/runtime/src/__tests__/provider-contract-matrix.test.ts
@@ -32,7 +32,7 @@ import {
import { PROVIDER_REGISTRY } from '@maka/core/llm-connections';
import { generateText, isStepCount, tool } from 'ai';
import { z } from 'zod';
-import { fetchProviderModels } from '../model-fetcher.js';
+import { discoverModels } from './model-discovery-fixture.js';
import { getAIModel } from '../model-factory.js';
import {
closeAllJsonServers,
@@ -180,7 +180,7 @@ async function runGeneratedDiscovery(
);
});
const connection = baseConnection(row, server.url);
- const models = await fetchProviderModels(connection, credentialCase.apiKey);
+ const models = await discoverModels(connection, credentialCase.apiKey);
if (handlerErrors.length > 0) throw handlerErrors[0];
assert.ok(requestCount >= 1, `${where} must request the model list`);
assert.deepEqual(
@@ -321,7 +321,7 @@ async function assertFallbackDiscoveryMakesNoRequest(row: ProviderContractRow):
respondJson(response, 500, { error: 'fallback discovery must not reach the network' });
});
const connection = baseConnection(row, server.url);
- const models = await fetchProviderModels(connection, API_KEY);
+ const models = await discoverModels(connection, API_KEY);
assert.equal(
requestCount,
0,
diff --git a/packages/runtime/src/__tests__/provider-contract-overrides.ts b/packages/runtime/src/__tests__/provider-contract-overrides.ts
index 13891c5d69..a8dda7c575 100644
--- a/packages/runtime/src/__tests__/provider-contract-overrides.ts
+++ b/packages/runtime/src/__tests__/provider-contract-overrides.ts
@@ -35,7 +35,7 @@ import assert from 'node:assert/strict';
import type { LlmConnection } from '@maka/core/llm-connections';
import { generateText, isStepCount, streamText, tool } from 'ai';
import { z } from 'zod';
-import { fetchProviderModels } from '../model-fetcher.js';
+import { discoverModels } from './model-discovery-fixture.js';
import { buildProviderOptions, getAIModel } from '../model-factory.js';
import { buildSubscriptionModelFetch } from '../subscription-model-fetch.js';
import {
@@ -349,7 +349,7 @@ async function runCloudflareDiscovery(): Promise<void> {
updatedAt: 1,
};
- assert.deepEqual(await fetchProviderModels(connection, 'cloudflare-test-token'), [
+ assert.deepEqual(await discoverModels(connection, 'cloudflare-test-token'), [
{ id: '@cf/meta/llama-text' },
]);
}
@@ -394,7 +394,7 @@ async function runGitHubCopilotDiscovery(): Promise<void> {
});
});
- const models = await fetchProviderModels(
+ const models = await discoverModels(
{
slug: 'github-copilot',
name: 'GitHub Copilot',
@@ -540,7 +540,7 @@ async function runGitHubCopilotWire(): Promise<void> {
createdAt: 1,
updatedAt: 1,
};
- const models = await fetchProviderModels(connection, 'github-account-token');
+ const models = await discoverModels(connection, 'github-account-token');
connection.models = models;
const modelFetch = buildSubscriptionModelFetch({
connection,
@@ -749,7 +749,7 @@ async function runFireworksDiscovery(): Promise<void> {
updatedAt: 1,
};
- const models = await fetchProviderModels(connection, 'fireworks-test-key');
+ const models = await discoverModels(connection, 'fireworks-test-key');
assert.deepEqual(models, [
{
id: 'accounts/acme/models/custom-agent',
@@ -881,7 +881,7 @@ async function assertOllamaModelContract(
};
assert.deepEqual(
- await fetchProviderModels(connection, ''),
+ await discoverModels(connection, ''),
discoveredModelIds.map((id) => ({ id })),
);
@@ -1009,7 +1009,7 @@ async function runCohereDiscovery(): Promise<void> {
updatedAt: 1,
};
- const models = await fetchProviderModels(connection, 'cohere-test-key');
+ const models = await discoverModels(connection, 'cohere-test-key');
assert.deepEqual(models, [
{ id: modelId, contextWindow: 128_000 },
{ id: 'command-a-reasoning-08-2025', contextWindow: 256_000 },
diff --git a/packages/runtime/src/model-fetcher.ts b/packages/runtime/src/model-fetcher.ts
index 03213ca944..eafe33e807 100644
--- a/packages/runtime/src/model-fetcher.ts
+++ b/packages/runtime/src/model-fetcher.ts
@@ -23,15 +23,13 @@ import {
effectiveBaseUrl,
isModelModality,
providerAuthSupportsApiKey,
- type LlmConnection,
type ModelInfo,
type ModelModality,
} from '@maka/core/llm-connections';
-import { generalizedErrorMessage } from '@maka/core/redaction';
import {
CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION,
CONNECTION_MODEL_ID_MAX_LENGTH,
- normalizeConnectionModelDiscoveryResult,
+ decodeConnectionModels,
} from '@maka/core/runtime-policy';
import { anthropicV1Url, googleApiUrl } from './provider-urls.js';
import { openAiCodexHeaders } from './subscription-auth.js';
@@ -44,7 +42,6 @@ import {
fetchForConnectionEffect,
type ConnectionEffectFetch,
type ConnectionEffectFetchDependency,
- type ConnectionEffectFetchOptions,
type ConnectionEffectResponse,
} from './connection-effect-fetch.js';
import {
@@ -138,28 +135,6 @@ type FireworksModelDiscovery = Extract<
{ kind: 'fireworks' }
>;
-export async function fetchProviderModels(
- connection: LlmConnection,
- apiKey: string,
- options: ConnectionEffectFetchOptions = {},
-): Promise<ModelInfo[]> {
- try {
- return normalizeDiscoveredModels(
- await fetchProviderModelsStrict(connection, apiKey, options.fetch),
- );
- } catch (error) {
- // Preserve status-bearing discovery errors so the sync layer can classify
- // auth/protocol/network failures; only wrap unknown errors for display.
- if (
- error instanceof OpenAiCodexDiscoveryError ||
- error instanceof ProviderModelDiscoveryHttpError
- ) {
- throw error;
- }
- throw new Error(generalizedErrorMessage(error, 'Failed to fetch provider models'));
- }
-}
-
export async function runConnectionModelDiscoveryEffect(
connection: ConnectionEffectConnection,
apiKey: string,
@@ -262,9 +237,6 @@ async function fetchProviderModelsStrict(
);
if (!r.ok) {
await r.cancel();
- if (connection.providerType === 'xai-oauth') {
- throw new ProviderModelDiscoveryHttpError(r.status);
- }
throw new ConnectionEffectHttpError(r.status);
}
const data = await readProviderJson<{ data?: unknown } | unknown[]>(r);
@@ -372,7 +344,7 @@ function cloudflareModelsUrl(baseUrl: string, page: number): string {
return url.toString();
}
-function normalizeDiscoveredModels(models: ModelInfo[]): ModelInfo[] {
+function normalizeConnectionEffectModels(models: ModelInfo[]): readonly ModelInfo[] {
const unique = new Map<string, ModelInfo>();
for (const model of models) {
if (typeof model?.id !== 'string') continue;
@@ -390,16 +362,8 @@ function normalizeDiscoveredModels(models: ModelInfo[]): ModelInfo[] {
throw new ConnectionEffectInvalidResponseError('Provider returned too many models');
}
}
- return [...unique.values()];
-}
-
-function normalizeConnectionEffectModels(models: ModelInfo[]): readonly ModelInfo[] {
try {
- return normalizeConnectionModelDiscoveryResult({
- models: normalizeDiscoveredModels(models),
- source: 'fetched',
- fetchedAt: 0,
- }).models;
+ return decodeConnectionModels([...unique.values()]);
} catch (error) {
throw new ConnectionEffectInvalidResponseError('Provider returned invalid model metadata', {
cause: error,
@@ -452,26 +416,6 @@ type RawOpenAiCodexModel = {
context_window?: unknown;
};
-/**
- * Discovery error carrying the HTTP status, so callers (syncOpenAiCodexConnection)
- * can classify auth failures (401/403) vs protocol errors (4xx) vs transient
- * network failures without string-matching the message.
- */
-export class OpenAiCodexDiscoveryError extends ConnectionEffectHttpError {
- constructor(status: number) {
- super(status);
- this.name = 'OpenAiCodexDiscoveryError';
- }
-}
-
-/** Structured status for standard provider `/models` endpoints. */
-export class ProviderModelDiscoveryHttpError extends ConnectionEffectHttpError {
- constructor(status: number) {
- super(status);
- this.name = 'ProviderModelDiscoveryHttpError';
- }
-}
-
/**
* Discover models from the ChatGPT/Codex OAuth backend
* (`chatgpt.com/backend-api/codex/models`). Unlike the public OpenAI API
@@ -481,7 +425,7 @@ export class ProviderModelDiscoveryHttpError extends ConnectionEffectHttpError {
* dropped; the rest are sorted by `priority` (ascending) to match the
* ChatGPT/Codex picker order.
*/
-export async function fetchOpenAiCodexModels(
+async function fetchOpenAiCodexModels(
baseUrl: string,
accessToken: string,
fetchFn?: ConnectionEffectFetch,
@@ -500,7 +444,7 @@ export async function fetchOpenAiCodexModels(
);
if (!response.ok) {
await response.cancel();
- throw new OpenAiCodexDiscoveryError(response.status);
+ throw new ConnectionEffectHttpError(response.status);
}
const payload = await readProviderJson<{ models?: unknown }>(response);
const models = providerObjectArray<RawOpenAiCodexModel>(
@@ -517,7 +461,7 @@ export async function fetchOpenAiCodexModels(
visible.sort((a, b) => priorityOfOpenAiCodexModel(a) - priorityOfOpenAiCodexModel(b));
return visible.map((model) => {
const entry: ModelInfo = { id: (model.slug as string).trim() };
- const contextWindow = contextWindowOfOpenAiCodexModel(model);
+ const contextWindow = providerTokenLimit(model.context_window);
if (contextWindow !== undefined) entry.contextWindow = contextWindow;
return entry;
});
@@ -529,12 +473,9 @@ function priorityOfOpenAiCodexModel(model: RawOpenAiCodexModel): number {
: 10_000;
}
-function contextWindowOfOpenAiCodexModel(model: RawOpenAiCodexModel): number | undefined {
- return typeof model.context_window === 'number' &&
- Number.isFinite(model.context_window) &&
- model.context_window > 0
- ? model.context_window
- : undefined;
+function providerTokenLimit(value: unknown): number | undefined {
+ // Invalid upstream metadata makes no claim about the model's limit.
+ return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : undefined;
}
function toGitHubCopilotModelInfo(model: RawGitHubCopilotModel): ModelInfo[] {
@@ -581,15 +522,16 @@ function toGitHubCopilotModelInfo(model: RawGitHubCopilotModel): ModelInfo[] {
limits?.vision?.supported_media_types?.some(
(type) => typeof type === 'string' && type.startsWith('image/'),
) === true;
- const contextWindow = limits?.max_context_window_tokens ?? limits?.max_prompt_tokens;
+ const contextWindow =
+ providerTokenLimit(limits?.max_context_window_tokens) ??
+ providerTokenLimit(limits?.max_prompt_tokens);
+ const maxOutputTokens = providerTokenLimit(limits?.max_output_tokens);
return [
{
id: model.id,
...(model.name ? { displayName: model.name } : {}),
- ...(typeof contextWindow === 'number' ? { contextWindow } : {}),
- ...(typeof limits?.max_output_tokens === 'number'
- ? { maxOutputTokens: limits.max_output_tokens }
- : {}),
+ ...(contextWindow === undefined ? {} : { contextWindow }),
+ ...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
apiProtocol,
capabilities: { vision, reasoning, functionCalling: true },
},
@@ -670,12 +612,11 @@ async function fetchCohereModels(
}
assertOptionalArray(model.endpoints, 'model endpoints');
if (!model.endpoints?.includes('chat')) return [];
+ const contextWindow = providerTokenLimit(model.context_length);
return [
{
id: model.name,
- ...(typeof model.context_length === 'number'
- ? { contextWindow: model.context_length }
- : {}),
+ ...(contextWindow === undefined ? {} : { contextWindow }),
},
];
}),
@@ -731,12 +672,11 @@ async function fetchFireworksModels(
authorization: `Bearer ${apiKey}`,
};
const signal = AbortSignal.timeout(MODEL_FETCH_TIMEOUT_MS);
+ let rawModelCount = 0;
const fetchPages = async <T extends object>(
path: string,
query: Readonly<Record<string, string>>,
itemKey: 'accounts' | 'models',
- maxItems: number,
- reserveItems?: (count: number) => void,
): Promise<T[]> => {
const items: T[] = [];
const seenPageTokens = new Set<string>();
@@ -768,10 +708,15 @@ async function fetchFireworksModels(
nextPageToken?: unknown;
}>(response);
const rawItems = providerObjectArray<T>(data[itemKey], `Fireworks ${itemKey}`);
- reserveItems?.(rawItems.length);
+ if (itemKey === 'models') {
+ rawModelCount += rawItems.length;
+ if (rawModelCount > CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION) {
+ throw new ConnectionEffectInvalidResponseError('Provider returned too many models');
+ }
+ }
items.push(...rawItems);
- if (items.length > maxItems) {
- throw new ConnectionEffectInvalidResponseError(`Provider returned too many ${itemKey}`);
+ if (itemKey === 'accounts' && items.length > FIREWORKS_MAX_ACCOUNTS) {
+ throw new ConnectionEffectInvalidResponseError('Provider returned too many accounts');
}
pageToken = nextProviderPageToken(data.nextPageToken);
if (!pageToken) return items;
@@ -786,7 +731,6 @@ async function fetchFireworksModels(
discovery.accountsPath,
{ pageSize: String(FIREWORKS_PAGE_SIZE) },
'accounts',
- FIREWORKS_MAX_ACCOUNTS,
);
const accountNames = [
...accounts.flatMap((account) =>
@@ -797,13 +741,6 @@ async function fetchFireworksModels(
if (accountNames.length > FIREWORKS_MAX_ACCOUNTS) {
throw new ConnectionEffectInvalidResponseError('Provider returned too many accounts');
}
- let rawModelCount = 0;
- const reserveModels = (count: number) => {
- rawModelCount += count;
- if (rawModelCount > CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION) {
- throw new ConnectionEffectInvalidResponseError('Provider returned too many models');
- }
- };
const modelLists: RawFireworksModel[][] = [];
for (let index = 0; index < accountNames.length; index += FIREWORKS_ACCOUNT_CONCURRENCY) {
modelLists.push(
@@ -811,13 +748,7 @@ async function fetchFireworksModels(
accountNames
.slice(index, index + FIREWORKS_ACCOUNT_CONCURRENCY)
.map((accountName) =>
- fetchPages<RawFireworksModel>(
- `/v1/${accountName}/models`,
- discovery.query,
- 'models',
- CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION,
- reserveModels,
- ),
+ fetchPages<RawFireworksModel>(`/v1/${accountName}/models`, discovery.query, 'models'),
),
)),
);
@@ -825,6 +756,7 @@ async function fetchFireworksModels(
return modelLists.flat().flatMap((model) => {
if (!model.name) return [];
+ const contextWindow = providerTokenLimit(model.contextLength);
const capabilities: NonNullable<ModelInfo['capabilities']> = {};
if (typeof model.supportsImageInput === 'boolean')
capabilities.vision = model.supportsImageInput;
@@ -834,7 +766,7 @@ async function fetchFireworksModels(
{
id: model.name,
...(model.displayName ? { displayName: model.displayName } : {}),
- ...(typeof model.contextLength === 'number' ? { contextWindow: model.contextLength } : {}),
+ ...(contextWindow === undefined ? {} : { contextWindow }),
...(Object.keys(capabilities).length ? { capabilities } : {}),
},
];
@@ -850,7 +782,9 @@ function toModelInfo(model: RawProviderModel): ModelInfo | null {
model.providers,
'model providers',
);
- const contextWindow = model.context_length ?? model.context_window;
+ const contextWindow =
+ providerTokenLimit(model.context_length) ?? providerTokenLimit(model.context_window);
+ const maxOutputTokens = providerTokenLimit(model.max_tokens);
const capabilities: NonNullable<ModelInfo['capabilities']> = {};
if (model.input_modalities?.includes('image')) capabilities.vision = true;
if (typeof model.capabilities?.reasoning === 'boolean')
@@ -885,8 +819,8 @@ function toModelInfo(model: RawProviderModel): ModelInfo | null {
return {
id: model.id,
...(model.display_name || model.name ? { displayName: model.display_name ?? model.name } : {}),
- ...(typeof contextWindow === 'number' ? { contextWindow } : {}),
- ...(typeof model.max_tokens === 'number' ? { maxOutputTokens: model.max_tokens } : {}),
+ ...(contextWindow === undefined ? {} : { contextWindow }),
+ ...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
...(Object.keys(capabilities).length ? { capabilities } : {}),
};
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org