You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maka.apache.org by as...@apache.org on 2026/09/15 01:51:47 UTC
(maka) 01/01: fix(eval): price Kimi K3 usage
This is an automated email from the ASF dual-hosted git repository.
Astro-Han pushed a commit to branch fix/eval-moonshot-metering
in repository https://gitbox.apache.org/repos/asf/maka.git
commit 655696e22d6a6339cb9a31446eed81080ed0f368
Author: AstroHan <le...@outlook.com>
AuthorDate: Tue Sep 15 09:51:28 2026 +0800
fix(eval): price Kimi K3 usage
Generated-by: Codex
---
.../eval/src/__tests__/external-subject.test.ts | 15 +++++-
.../src/__tests__/lifecycle-boundaries.test.ts | 18 ++++++-
.../provider-admission-integration.test.ts | 2 +-
packages/eval/src/maka-subject.ts | 12 ++---
packages/eval/src/provider-metering.ts | 55 ++++++++++++++++++----
5 files changed, 79 insertions(+), 23 deletions(-)
diff --git a/packages/eval/src/__tests__/external-subject.test.ts b/packages/eval/src/__tests__/external-subject.test.ts
index d5ee0e8473..5e58f31d63 100644
--- a/packages/eval/src/__tests__/external-subject.test.ts
+++ b/packages/eval/src/__tests__/external-subject.test.ts
@@ -122,7 +122,7 @@ test('a checkpoint the proxy never settled is a lower bound however complete it
admittedRequests: 1,
usageRequests: 1,
removedWebTools: 0,
- models: [],
+ models: ['kimi-k3'],
toolNames: [],
};
try {
@@ -151,7 +151,18 @@ test('a checkpoint the proxy never settled is a lower bound however complete it
);
assert.equal(settled?.artifact.usageComplete, true);
assert.equal(settled?.artifact.tokenBasis, 'complete');
- assert.ok((settled?.costUsd ?? 0) > 0);
+ assert.equal(settled?.costUsd, 0.000384);
+
+ await writeSignedCheckpoint(join(trialPath, 'agent/codex.provider-usage.json'), {
+ ...checkpoint,
+ settled: true,
+ models: ['kimi-k3', 'unknown-model'],
+ });
+ assert.equal(
+ (await recoverExternalMetering({ trialPath, meteringSecret: METERING_SECRET }, 'codex'))
+ ?.costUsd,
+ null,
+ );
} finally {
await rm(root, { recursive: true, force: true });
}
diff --git a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts
index fe84a2d6ec..85f77d077a 100644
--- a/packages/eval/src/__tests__/lifecycle-boundaries.test.ts
+++ b/packages/eval/src/__tests__/lifecycle-boundaries.test.ts
@@ -504,6 +504,21 @@ test('Maka framework termination is authoritative before stdout decoding', async
{ kind: 'subject-reported-cost', costUsd: reported },
);
+ const kimi = await executeMaka(
+ 'exited',
+ (executionId) =>
+ JSON.stringify({
+ executionId,
+ kind: 'settled',
+ status: 'completed',
+ usage: usage(),
+ costUsd: reported,
+ }),
+ undefined,
+ 'kimi-k3',
+ );
+ assert.equal(kimi.costUsd, 0.0001299);
+
const external = await createExternalSubjectAdapter().execute({
cell: cell('external', { command: '/opt/competitor', args: [], result: 'exit-code' }),
context: {
@@ -1276,9 +1291,10 @@ async function executeMaka(
termination: 'exited' | 'framework_timeout',
stdout: (executionId: string) => string,
diagnostic?: Awaited<ReturnType<SubjectExecutionContext['execute']>>['diagnostic'],
+ model = 'deepseek-v4-flash',
) {
return createMakaSubjectAdapter().execute({
- cell: cell('maka', makaConfig()),
+ cell: cell('maka', { ...makaConfig(), model }),
context: {
cwd: '/app',
taskInput: 'solve',
diff --git a/packages/eval/src/__tests__/provider-admission-integration.test.ts b/packages/eval/src/__tests__/provider-admission-integration.test.ts
index 159b8dd829..f96f1f03f8 100644
--- a/packages/eval/src/__tests__/provider-admission-integration.test.ts
+++ b/packages/eval/src/__tests__/provider-admission-integration.test.ts
@@ -158,7 +158,7 @@ test('provider failures remain infrastructure failures until inference admission
const child = join(root, 'child.mjs');
await writeFile(
child,
- "if(process.env.OPENAI_API_KEY||process.env.ANTHROPIC_API_KEY||process.env.DEEPSEEK_API_KEY!=='maka-eval-local'||process.env.MAKA_EVAL_RESULT_TOKEN)process.exit(9);await fetch(`${process.env.DEEPSEEK_BASE_URL}/responses`,{method:'POST',body:'{}'});console.error('stderr-sentinel-must-not-persist');console.log('stdout-sentinel-must-not-persist');console.log(JSON.stringify({type:'error'}));process.exit(1);\n",
+ "if(process.env.OPENAI_API_KEY||process.env.ANTHROPIC_API_KEY||process.env.DEEPSEEK_API_KEY!=='maka-eval-local'||process.env.MAKA_EVAL_RESULT_TOKEN)process.exit(9);await fetch(`${process.env.DEEPSEEK_BASE_URL}/responses`,{method:'POST',body:'{\"model\":\"deepseek-v4-flash\"}'});console.error('stderr-sentinel-must-not-persist');console.log('stdout-sentinel-must-not-persist');console.log(JSON.stringify({type:'error'}));process.exit(1);\n",
);
try {
const wrapper = new URL('../harbor-external-subject.js', import.meta.url);
diff --git a/packages/eval/src/maka-subject.ts b/packages/eval/src/maka-subject.ts
index a76b4305b3..49df2d7ab1 100644
--- a/packages/eval/src/maka-subject.ts
+++ b/packages/eval/src/maka-subject.ts
@@ -27,8 +27,7 @@ import {
MAKA_SUBJECT_STDERR_PATH,
MAKA_SUBJECT_STDOUT_PATH,
} from './maka-artifacts.js';
-import { deepSeekCostUsd } from './provider-metering.js';
-import type { NormalizedUsage } from './result.js';
+import { modelCostUsd } from './provider-metering.js';
import type { SubjectAdapter, SubjectExecutionContext } from './runner.js';
export function createMakaSubjectAdapter(): SubjectAdapter {
@@ -83,7 +82,7 @@ export function createMakaSubjectAdapter(): SubjectAdapter {
const settled = projection?.kind === 'settled' ? projection : undefined;
return {
usage: settled?.usage ?? null,
- costUsd: settled ? estimateDeepSeekCost(settled.usage, config.model) : null,
+ costUsd: settled ? modelCostUsd(settled.usage, config.model) : null,
durationMs: Date.now() - startedAt,
status: 'failed' as const,
failureReason: 'Maka subject exceeded the framework timeout',
@@ -131,7 +130,7 @@ export function createMakaSubjectAdapter(): SubjectAdapter {
// billed at the runtime's rates and the others at this package's --
// a difference in the reported figure that no agent behaviour caused.
// It is kept as evidence below rather than used as the authority.
- costUsd: estimateDeepSeekCost(projection.usage, config.model),
+ costUsd: modelCostUsd(projection.usage, config.model),
durationMs: Date.now() - startedAt,
status,
failureReason,
@@ -328,8 +327,3 @@ function positive(value: unknown, where: string): number {
if (!Number.isSafeInteger(value) || (value as number) < 1) throw new Error(`${where} is invalid`);
return value as number;
}
-
-function estimateDeepSeekCost(usage: NormalizedUsage, model: string): number | null {
- if (model !== 'deepseek-v4-flash') return null;
- return deepSeekCostUsd(usage);
-}
diff --git a/packages/eval/src/provider-metering.ts b/packages/eval/src/provider-metering.ts
index 3113b2c0d4..bf5d87ecda 100644
--- a/packages/eval/src/provider-metering.ts
+++ b/packages/eval/src/provider-metering.ts
@@ -69,21 +69,25 @@ export function deriveMetering(counts: ProviderMeteringCounts): DerivedMetering
counts.settled &&
counts.admittedRequests > 0 &&
counts.usageRequests === counts.admittedRequests;
+ const models = [...new Set(counts.models)];
return {
settledRequests: counts.requests - counts.inFlightRequests,
missingUsageRequests: counts.admittedRequests - counts.usageRequests,
usageComplete,
- costUsd: counts.usage && usageComplete ? deepSeekCostUsd(counts.usage) : null,
+ costUsd:
+ counts.usage && usageComplete && models.length === 1
+ ? modelCostUsd(counts.usage, models[0]!)
+ : null,
tokenBasis: usageComplete ? 'complete' : 'lower-bound',
};
}
// Published DeepSeek V4 Flash prices, in USD per million tokens. Eval bills
-// every arm from this table rather than from each arm's own transcription of
-// it, so a difference in reported cost can only come from a difference in what
-// the agent spent. Pi is additionally handed the table for its own display; the
-// other frameworks are given no cost block at all, which is why their figures
-// are not consulted here.
+// from its model table rather than from each subject's own transcription, so a
+// difference in reported cost can only come from a difference in what the agent
+// spent. Pi is additionally handed this entry for its own display; the other
+// frameworks are given no cost block, which is why their figures are not
+// consulted here.
//
// Rates as published 2026-08-17, normalized at the off-peak band: DeepSeek
// bills peak hours (01:00–04:00 and 06:00–10:00 UTC) at exactly double these,
@@ -102,16 +106,47 @@ export const DEEPSEEK_V4_FLASH_COST = {
cacheWrite: 0.22,
} as const;
+// International Kimi API prices published 2026-09-15. Kimi's automatic cache
+// has hit and miss rates but no separate write rate, so a reported cache write
+// remains an input miss just as it does for DeepSeek.
+const KIMI_K3_COST = {
+ input: 3,
+ output: 15,
+ cacheRead: 0.3,
+ cacheWrite: 3,
+} as const;
+
+const MODEL_COSTS: Readonly<Record<string, ModelCost>> = {
+ 'deepseek-v4-flash': DEEPSEEK_V4_FLASH_COST,
+ 'kimi-k3': KIMI_K3_COST,
+};
+
// `inputTokens` is normalized to include both cached kinds, so each kind is
// subtracted out and charged at its own rate. Leaving cache writes subtracted
// but unpriced billed them at zero.
export function deepSeekCostUsd(usage: ProviderUsage): number {
+ return costUsd(usage, DEEPSEEK_V4_FLASH_COST);
+}
+
+export function modelCostUsd(usage: ProviderUsage, model: string): number | null {
+ const cost = MODEL_COSTS[model];
+ return cost ? costUsd(usage, cost) : null;
+}
+
+interface ModelCost {
+ readonly input: number;
+ readonly output: number;
+ readonly cacheRead: number;
+ readonly cacheWrite: number;
+}
+
+function costUsd(usage: ProviderUsage, cost: ModelCost): number {
const uncached = Math.max(0, usage.inputTokens - usage.cacheReadTokens - usage.cacheWriteTokens);
return (
- (uncached * DEEPSEEK_V4_FLASH_COST.input +
- usage.cacheReadTokens * DEEPSEEK_V4_FLASH_COST.cacheRead +
- usage.cacheWriteTokens * DEEPSEEK_V4_FLASH_COST.cacheWrite +
- usage.outputTokens * DEEPSEEK_V4_FLASH_COST.output) /
+ (uncached * cost.input +
+ usage.cacheReadTokens * cost.cacheRead +
+ usage.cacheWriteTokens * cost.cacheWrite +
+ usage.outputTokens * cost.output) /
1_000_000
);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org