You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maka.apache.org by as...@apache.org on 2026/09/01 14:19:33 UTC
(maka) 03/09: feat(scripts): report models.dev snapshot drift against upstream
This is an automated email from the ASF dual-hosted git repository.
Astro-Han pushed a commit to branch ci/4398-model-metadata-drift-check
in repository https://gitbox.apache.org/repos/asf/maka.git
commit 6ce0f8accadc6a45c69b67295f3f9c33b69c99fe
Author: AstroHan <le...@outlook.com>
AuthorDate: Tue Sep 1 18:19:06 2026 +0800
feat(scripts): report models.dev snapshot drift against upstream
`check:model-metadata` proves the generated modules match the committed
snapshot. Nothing proved the snapshot still matches models.dev, so the
snapshot could sit weeks behind upstream with every check green — which
is exactly what happened: models.dev listed glm-5.3 from 2026-08-14 and
the snapshot regenerated on 2026-08-29 without it.
`--drift` fetches upstream and compares model by model, printing what
differs and exiting non-zero. Per model rather than per projection on
purpose: a refresh aborts on the first shape it cannot project, so one
surprising model would otherwise hide every other difference. A rejected
shape becomes its own bucket instead.
The mode never writes, so it is safe to run against a checkout that must
stay clean. It stays out of check:asf-source and check:release: both run
on every affected pull request, and neither should need models.dev to be
reachable.
Generated-by: Claude Code
---
package.json | 1 +
scripts/sync-model-metadata.mjs | 130 +++++++++++++++++++++++++++++++----
scripts/sync-model-metadata.test.mjs | 98 ++++++++++++++++++++++++++
3 files changed, 216 insertions(+), 13 deletions(-)
diff --git a/package.json b/package.json
index 08a3131b68..ccbba4bcdb 100644
--- a/package.json
+++ b/package.json
@@ -92,6 +92,7 @@
"sync:model-metadata": "node scripts/sync-model-metadata.mjs",
"refresh:model-metadata": "node scripts/sync-model-metadata.mjs --refresh",
"check:model-metadata": "node scripts/sync-model-metadata.mjs --check",
+ "check:model-metadata-drift": "node scripts/sync-model-metadata.mjs --drift",
"generate:bundled-skills": "node scripts/gen-bundled-skill-catalog.mjs",
"computer-use": "node scripts/computer-use.mjs",
"windows:inventory": "node --test scripts/windows-test-inventory.test.mjs && node scripts/windows-test-inventory.mjs --check",
diff --git a/scripts/sync-model-metadata.mjs b/scripts/sync-model-metadata.mjs
index 02782434d4..8f820cdf95 100644
--- a/scripts/sync-model-metadata.mjs
+++ b/scripts/sync-model-metadata.mjs
@@ -117,11 +117,22 @@ export async function main(argv = process.argv) {
(outputPath === DEFAULT_OUTPUT ? DEFAULT_PRICING_OUTPUT : undefined);
const refresh = argv.includes('--refresh');
const check = argv.includes('--check');
+ const drift = argv.includes('--drift');
const acceptUpstreamRemovals = argv.includes('--accept-upstream-removals');
- if (refreshInputPath && !refresh) throw new Error('--refresh-input requires --refresh');
+ if (drift && (refresh || check)) {
+ throw new Error('--drift reports without writing and cannot combine with --refresh or --check');
+ }
+ if (refreshInputPath && !refresh && !drift) {
+ throw new Error('--refresh-input requires --refresh or --drift');
+ }
if (acceptUpstreamRemovals && !refresh) {
throw new Error('--accept-upstream-removals requires --refresh');
}
+ if (drift) {
+ const report = await collectDrift(await loadSnapshot(snapshotPath), refreshInputPath);
+ process.stdout.write(`${formatDrift(report)}\n`);
+ return report;
+ }
const source = refresh
? await refreshSnapshot(snapshotPath, refreshInputPath, { acceptUpstreamRemovals })
@@ -235,19 +246,25 @@ function buildProjection(catalog) {
return { metadata, pricing, providerFacts, providerOverrides };
}
-async function refreshSnapshot(snapshotPath, refreshInputPath, options = {}) {
- let sourceText;
- let sourceEtag = null;
- let retrievedAt = new Date().toISOString();
+async function readUpstream(refreshInputPath) {
if (refreshInputPath) {
- sourceText = await readFile(refreshInputPath, 'utf8');
- } else {
- const response = await fetch(SOURCE_URL, { signal: AbortSignal.timeout(10_000) });
- if (!response.ok) throw new Error(`models.dev returned HTTP ${response.status}`);
- sourceText = await response.text();
- sourceEtag = response.headers.get('etag');
- retrievedAt = new Date(response.headers.get('date') ?? Date.now()).toISOString();
+ return {
+ text: await readFile(refreshInputPath, 'utf8'),
+ etag: null,
+ retrievedAt: new Date().toISOString(),
+ };
}
+ const response = await fetch(SOURCE_URL, { signal: AbortSignal.timeout(10_000) });
+ if (!response.ok) throw new Error(`models.dev returned HTTP ${response.status}`);
+ return {
+ text: await response.text(),
+ etag: response.headers.get('etag'),
+ retrievedAt: new Date(response.headers.get('date') ?? Date.now()).toISOString(),
+ };
+}
+
+async function refreshSnapshot(snapshotPath, refreshInputPath, options = {}) {
+ const { text: sourceText, etag: sourceEtag, retrievedAt } = await readUpstream(refreshInputPath);
const projection = buildProjection(selectCatalog(JSON.parse(sourceText)));
if (!options.acceptUpstreamRemovals) {
const previous = await loadSnapshotIfPresent(snapshotPath);
@@ -339,6 +356,91 @@ function projectionPath(path) {
return `/${path.map((segment) => String(segment).replaceAll('~', '~0').replaceAll('/', '~1')).join('/')}`;
}
+// `--check` only proves the generated modules match the committed snapshot.
+// Nothing compared that snapshot against models.dev, which is how it stayed
+// weeks behind upstream without anything reporting it. This walks the two one
+// model at a time, so a shape the projector rejects becomes its own finding
+// instead of aborting the whole comparison the way a refresh does.
+const DRIFT_LIST_LIMIT = 20;
+
+async function collectDrift(snapshot, refreshInputPath) {
+ const catalog = JSON.parse((await readUpstream(refreshInputPath)).text);
+ const previousMetadata = snapshot.projection.metadata;
+ const previousPricing = new Map(
+ snapshot.projection.pricing.map((entry) => [entry.modelKey, entry]),
+ );
+ const report = { missingProviders: [], unprojectable: [], added: [], removed: [], changed: [] };
+ for (const [providerType, sourceId] of Object.entries(PROVIDERS)) {
+ const provider = catalog[sourceId];
+ const previousModels = previousMetadata[providerType] ?? {};
+ const models = provider?.models;
+ if (!models || typeof models !== 'object' || Array.isArray(models)) {
+ report.missingProviders.push(`${providerType} (models.dev ${sourceId})`);
+ continue;
+ }
+ const priced = !PRICING_EXCLUDED_PROVIDER_TYPES.has(providerType);
+ const ids = [...new Set([...Object.keys(previousModels), ...Object.keys(models)])].sort();
+ for (const id of ids) {
+ const model = models[id];
+ const previous = previousModels[id];
+ if (model === undefined) {
+ report.removed.push(`${providerType}/${id}`);
+ continue;
+ }
+ let metadata;
+ let pricing;
+ try {
+ metadata = toMetadata(sourceId, id, provider, model);
+ pricing = priced ? toPricing(providerType, id, model) : undefined;
+ } catch (error) {
+ report.unprojectable.push(`${providerType}/${id}: ${error.message}`);
+ continue;
+ }
+ if (previous === undefined) {
+ report.added.push(`${providerType}/${id}`);
+ continue;
+ }
+ const fields = driftedFields(previous, metadata);
+ if (priced && !sameValue(previousPricing.get(`${providerType}:${id}`), pricing)) {
+ fields.push('pricing');
+ }
+ if (fields.length > 0) report.changed.push(`${providerType}/${id}: ${fields.join(', ')}`);
+ }
+ }
+ const drifted = Object.values(report).some((entries) => entries.length > 0);
+ return { ...report, drifted };
+}
+
+function driftedFields(previous, next) {
+ const keys = [...new Set([...Object.keys(previous), ...Object.keys(next)])].sort();
+ return keys.filter((key) => !sameValue(previous[key], next[key]));
+}
+
+// Both sides are projector output, so their keys are already in one order.
+function sameValue(left, right) {
+ return JSON.stringify(left) === JSON.stringify(right);
+}
+
+function formatDrift(report) {
+ const lines = [];
+ for (const [label, entries] of [
+ ['providers missing upstream', report.missingProviders],
+ ['models the projector rejects', report.unprojectable],
+ ['models upstream has and the snapshot does not', report.added],
+ ['models the snapshot has and upstream does not', report.removed],
+ ['models whose projection changed', report.changed],
+ ]) {
+ if (entries.length === 0) continue;
+ lines.push(`${label}: ${entries.length}`);
+ for (const entry of entries.slice(0, DRIFT_LIST_LIMIT)) lines.push(` ${entry}`);
+ if (entries.length > DRIFT_LIST_LIMIT) {
+ lines.push(` ... and ${entries.length - DRIFT_LIST_LIMIT} more`);
+ }
+ }
+ if (!report.drifted) return `${SOURCE_URL} matches the committed snapshot.`;
+ return [`${SOURCE_URL} has drifted from the committed snapshot.`, ...lines].join('\n');
+}
+
async function replaceFilesTransactionally(writes) {
if (new Set(writes.map((write) => write.path)).size !== writes.length) {
throw new Error('model metadata outputs must use distinct paths');
@@ -705,7 +807,9 @@ function buildPricingModule(pricing, source) {
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
- await main();
+ // Drift is a finding, not a crash: the report has already been printed, and
+ // the exit code is what a scheduled job branches on.
+ if ((await main())?.drifted) process.exitCode = 1;
}
function option(name, argv) {
diff --git a/scripts/sync-model-metadata.test.mjs b/scripts/sync-model-metadata.test.mjs
index b0c9f0eef3..cf63c04183 100644
--- a/scripts/sync-model-metadata.test.mjs
+++ b/scripts/sync-model-metadata.test.mjs
@@ -302,6 +302,104 @@ test('refresh rejects unknown model modalities instead of dropping them', async
}
});
+// The drift report goes to stdout for the scheduled job's summary; tests read
+// the returned buckets instead of the printed text.
+async function drift(argv) {
+ const written = [];
+ const original = process.stdout.write;
+ process.stdout.write = (chunk) => {
+ written.push(String(chunk));
+ return true;
+ };
+ try {
+ return { report: await main(['node', 'sync-model-metadata.mjs', '--drift', ...argv]) };
+ } finally {
+ process.stdout.write = original;
+ assert.ok(written.join('').length > 0, 'drift must print a report');
+ }
+}
+
+test('drift finds nothing when the snapshot still matches upstream', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'maka-model-drift-clean-'));
+ try {
+ const input = join(root, 'api.json');
+ const snapshot = join(root, 'snapshot.json');
+ await writeFile(input, JSON.stringify(fixtureCatalog()));
+ await main([
+ 'node',
+ 'sync-model-metadata.mjs',
+ '--refresh',
+ '--refresh-input',
+ input,
+ '--snapshot',
+ snapshot,
+ '--output',
+ join(root, 'metadata.ts'),
+ ]);
+
+ const { report } = await drift(['--snapshot', snapshot, '--refresh-input', input]);
+ assert.equal(report.drifted, false);
+ assert.deepEqual(
+ [report.added, report.removed, report.changed, report.unprojectable, report.missingProviders],
+ [[], [], [], [], []],
+ );
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test('drift separates a rejected shape from a real upstream difference', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'maka-model-drift-'));
+ try {
+ const committed = join(root, 'api.json');
+ const upstream = join(root, 'upstream.json');
+ const snapshot = join(root, 'snapshot.json');
+ const before = fixtureCatalog();
+ before.anthropic.models.legacy = { ...before.anthropic.models.model, name: 'Legacy' };
+ await writeFile(committed, JSON.stringify(before));
+ await main([
+ 'node',
+ 'sync-model-metadata.mjs',
+ '--refresh',
+ '--refresh-input',
+ committed,
+ '--snapshot',
+ snapshot,
+ '--output',
+ join(root, 'metadata.ts'),
+ ]);
+
+ const after = JSON.parse(JSON.stringify(before));
+ delete after.anthropic.models.legacy;
+ after.anthropic.models.added = { ...after.anthropic.models.model, name: 'Added' };
+ after.anthropic.models.model.name = 'Renamed';
+ after.anthropic.models.model.cost = { input: 9, output: 9 };
+ after.groq.models.model.modalities = { input: ['text', 'hologram'], output: ['text'] };
+ delete after.openai;
+ await writeFile(upstream, JSON.stringify(after));
+
+ const { report } = await drift(['--snapshot', snapshot, '--refresh-input', upstream]);
+ assert.equal(report.drifted, true);
+ assert.deepEqual(report.added, ['anthropic/added']);
+ assert.deepEqual(report.removed, ['anthropic/legacy']);
+ assert.deepEqual(report.changed, ['anthropic/model: displayName, pricing']);
+ assert.deepEqual(report.missingProviders, ['openai (models.dev openai)']);
+ assert.equal(report.unprojectable.length, 1);
+ assert.match(report.unprojectable[0], /^groq\/model: .*unsupported modalities/u);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test('drift refuses to combine with the modes that write', async () => {
+ for (const mode of ['--refresh', '--check']) {
+ await assert.rejects(
+ main(['node', 'sync-model-metadata.mjs', '--drift', mode]),
+ /--drift reports without writing/u,
+ );
+ }
+});
+
test('refresh carries the video and pdf modalities models.dev declares', async () => {
const root = await mkdtemp(join(tmpdir(), 'maka-model-snapshot-video-'));
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org