You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2026/10/17 06:22:52 UTC
(camel-performance-tests) branch main updated (927fdb0 -> fe437d5)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
from 927fdb0 ai-benchmark: the ladder keeps values in variables, not exchange properties
new c46e40a ai-benchmark: BENCH_FAST waits for what a step checks instead of fixed sleeps, and traces the tool times
new 4fae7f9 ai-benchmark: BENCH_FAST counts only the errors after a step began, BENCH_SERVER_INSTRUCTIONS, and data dirs for the ladder
new d889c91 ai-benchmark: 25 tool calls per step instead of 20
new f262f3e ai-benchmark: the server restart stops only its own server, and the postgres wait reads the new JSON errors
new 6036d63 ai-benchmark: checks accept quoted values, and the apps scan every 500 ms like camel_run
new fe437d5 ai-benchmark: camel_catalog_doc answers are cut at 16000 characters, and the trace records each answer's length
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
ai-benchmark/agent_local.py | 5 +-
ai-benchmark/agent_mcp_stepwise.py | 175 ++++++++++++++++++++++++++++++++-----
ai-benchmark/gen_stepwise.py | 15 ++--
ai-benchmark/start-server.sh | 7 +-
4 files changed, 170 insertions(+), 32 deletions(-)
(camel-performance-tests) 05/06: ai-benchmark: checks accept quoted values, and the apps scan every 500 ms like camel_run
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit 6036d633f29526b39d3dc137a3941562b008185b
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Fri Oct 16 22:48:57 2026 +0200
ai-benchmark: checks accept quoted values, and the apps scan every 500 ms like camel_run
Three file checks failed a right answer written with quotes: move: "done/${file:name}" (4 times in r14a),
throwExceptionOnFailure: "false" and parallelProcessing: "true". They accept single and double quotes now.
The harness starts the apps itself with camel run --source-dir --dev; it now passes the optional
camel.main.?routesReloadInterval=500 that camel_run passes since CAMEL-25661, so an agent's writes reload as soon as
they do with camel_run.
Co-Authored-By: Claude Opus 5.5 (1M context) <no...@anthropic.com>
Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m
---
ai-benchmark/agent_mcp_stepwise.py | 5 ++++-
ai-benchmark/gen_stepwise.py | 6 +++---
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py
index 70b2d0f..bcf3d51 100755
--- a/ai-benchmark/agent_mcp_stepwise.py
+++ b/ai-benchmark/agent_mcp_stepwise.py
@@ -527,7 +527,10 @@ def main():
# them); the MCP tools find the integration by name
name = os.path.basename(project)
run_out = open(os.path.join(OUT, "camel-run.out"), "w")
- proc = subprocess.Popen(["camel", "run", "--source-dir=" + project, "--dev", "--name=" + name, "--logging-color=false"],
+ # the scan interval camel_run uses since CAMEL-25661; the ? makes it optional, so a Camel without the option
+ # logs a WARN and starts
+ proc = subprocess.Popen(["camel", "run", "--source-dir=" + project, "--dev", "--name=" + name, "--logging-color=false",
+ "--prop=camel.main.?routesReloadInterval=500"],
stdout=run_out, stderr=subprocess.STDOUT, cwd=project)
for _ in range(60):
time.sleep(1)
diff --git a/ai-benchmark/gen_stepwise.py b/ai-benchmark/gen_stepwise.py
index 4b5eb65..78f161a 100644
--- a/ai-benchmark/gen_stepwise.py
+++ b/ai-benchmark/gen_stepwise.py
@@ -300,7 +300,7 @@ EXAMPLES.append({
"check": {"file_regex": "multicast", "log_regex": ["Warehouse: pick 2 line\\(s\\) for ORD-1001", "Invoicing: bill customer C-482 for ORD-1001"], "log_not_regex": "(Warehouse|Invoicing).*ORD-1003"},
"reference": {FM: fm_s3}},
{"request": "Add parallelProcessing: true to the multicast so both departments get the order at the same moment.",
- "check": {"file_regex": "parallelProcessing:\\s*true", "log_regex": ["Warehouse: pick 2 line\\(s\\) for ORD-1001", "Invoicing: bill customer C-482 for ORD-1001"]},
+ "check": {"file_regex": "parallelProcessing:\\s*[\"']?true", "log_regex": ["Warehouse: pick 2 line\\(s\\) for ORD-1001", "Invoicing: bill customer C-482 for ORD-1001"]},
"reference": {FM: fm_s4}},
]})
@@ -584,7 +584,7 @@ EXAMPLES.append({
"initial": {FP: fp_s1, "application.properties": DATA_DIR + "samples,inbox,archive\n"},
"steps": [
{"request": "The invoices route must only take the JSON files: include .*\\.json, and move each processed file to the done directory instead of the default .camel directory. The driver's note note-2005.txt must stay untouched in inbox.",
- "check": {"file_regex": "move: done", "file_regex2": "include", "log_regex": "Received invoice-2004.json", "log_not_regex": "Received note-2005", "files": {"inbox/note-2005.txt": "Handwritten note", "done/invoice-2001.json": "INV-2001"}},
+ "check": {"file_regex": "move: [\"']?done", "file_regex2": "include", "log_regex": "Received invoice-2004.json", "log_not_regex": "Received note-2005", "files": {"inbox/note-2005.txt": "Handwritten note", "done/invoice-2001.json": "INV-2001"}},
"reference": {FP: fp_s2}},
{"request": "Unmarshal each invoice from JSON with Jackson and log \"Invoice ${body[invoiceId]} for ${body[orderId]}: ${body[amount]} EUR\" instead of the file name.",
"check": {"file_regex": "unmarshal", "log_regex": ["Invoice INV-2001 for ORD-1001: 49.4 EUR", "Invoice INV-2003 for ORD-1003: -5.0 EUR"]},
@@ -675,7 +675,7 @@ EXAMPLES.append({
"check": {"file_regex": "toD", "log_regex": ["ORD-1001: CAMEL-TSHIRT x 2, 120 in stock", "ORD-1003: CAMEL-CAP x 1, 0 in stock"]},
"reference": {HC: hc_s3}},
{"request": "Decide per line: add throwExceptionOnFailure=false to the HTTP call so a 404 does not throw, then a choice: when the header CamelHttpResponseCode is not 200 log \"${variable.orderId}: ${body[error]} (HTTP ${header.CamelHttpResponseCode})\", when ${body[qty]} >= ${variable.needed} log \"${variable.orderId}: ${variable.sku} x ${variable.needed}, ${body[qty]} in stock, ok\", otherwise log \"${variable.orderId}: ${variable.sku} x ${variable.needed}, only ${body[qty]} in [...]
- "check": {"file_regex": "throwExceptionOnFailure[=:] *false", "log_regex": ["ORD-1001: CAMEL-TSHIRT x 2, 120 in stock, ok", "ORD-1003: CAMEL-CAP x 1, only 0 in stock, back-order"]},
+ "check": {"file_regex": "throwExceptionOnFailure[=:] *[\"']?false", "log_regex": ["ORD-1001: CAMEL-TSHIRT x 2, 120 in stock, ok", "ORD-1003: CAMEL-CAP x 1, only 0 in stock, back-order"]},
"reference": {HC: hc_s4}},
]})
(camel-performance-tests) 06/06: ai-benchmark: camel_catalog_doc answers are cut at 16000 characters, and the trace records each answer's length
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit fe437d55c152a9e2f397045b9ab124dc96956304
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Fri Oct 16 23:00:41 2026 +0200
ai-benchmark: camel_catalog_doc answers are cut at 16000 characters, and the trace records each answer's length
At 6000 the model never saw most of a component's options: for file, include, move and sortBy sit past 11000
characters, and in r14b the model asked for all of them 75 times to get the same first 6000. 16000 is the longest
answer camel_catalog_doc gives since CAMEL-25541; the other tools stay at 6000, since the harness keeps every answer
of a step in the conversation. BENCH_CATALOG_CAP=6000 gives the r1-r14 behaviour. The trace records chars (the
answer) and shown (what the model got).
Co-Authored-By: Claude Opus 5.5 (1M context) <no...@anthropic.com>
Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m
---
ai-benchmark/agent_mcp_stepwise.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py
index bcf3d51..df46d34 100755
--- a/ai-benchmark/agent_mcp_stepwise.py
+++ b/ai-benchmark/agent_mcp_stepwise.py
@@ -40,6 +40,10 @@ TOOL_GROUPS = os.environ.get("BENCH_TOOL_GROUPS") == "1"
# invalid YAML is still refused); what the model knows by itself
BARE = os.environ.get("BENCH_BARE") == "1"
TOOL_RESULT_CAP = 6000
+# camel_catalog_doc answers are cut later: at 6000 the model never saw most of a component's options (file: include,
+# move and sortBy sit past 11000 characters), and r14b asked for all of them 75 times to get the same first 6000. 16000
+# is the longest answer camel_catalog_doc gives since CAMEL-25541. BENCH_CATALOG_CAP=6000 gives the r1-r14 behaviour
+CATALOG_RESULT_CAP = int(os.environ.get("BENCH_CATALOG_CAP", "16000"))
SHARED = ["camel_catalog_doc", "camel_catalog_find", "camel_catalog_sample", "camel_validate_source", "camel_get_files", "camel_write_file",
"camel_edit_file",
@@ -627,8 +631,9 @@ def main():
refused += 1
elif fn["name"] in ("camel_write_file", "camel_edit_file"):
before["last_write"] = error_snapshot(mcp, name)
- out = out[:TOOL_RESULT_CAP]
- trace.write(json.dumps({"tool": fn["name"], "ms": ms, "args": {k: (v if k != "content" else v[:1500]) for k, v in args.items()}, "result": out[:800]}) + "\n"); trace.flush()
+ chars = len(out)
+ out = out[:CATALOG_RESULT_CAP if fn["name"] == "camel_catalog_doc" else TOOL_RESULT_CAP]
+ trace.write(json.dumps({"tool": fn["name"], "ms": ms, "args": {k: (v if k != "content" else v[:1500]) for k, v in args.items()}, "result": out[:800], "chars": chars, "shown": len(out)}) + "\n"); trace.flush()
messages.append({"role": "tool", "content": out, "tool_name": fn["name"]})
continue
answer = msg.get("content") or ""
(camel-performance-tests) 01/06: ai-benchmark: BENCH_FAST waits for what a step checks instead of fixed sleeps, and traces the tool times
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit c46e40a0925db965c7a9b9be4c71f8e77f5df44b
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Wed Oct 14 20:45:40 2026 +0200
ai-benchmark: BENCH_FAST waits for what a step checks instead of fixed sleeps, and traces the tool times
22 of a run's 55 minutes were fixed sleeps (Oct 14 2026). The sleeps are wait_seconds after each check (11 min), 6 s after
each reference write (7 min), reload polls every 2 s and a 3 s sleep between examples. Dev mode has scanned every 2 s since
CAMEL-25032, so the waits padded for a delay that is gone. With BENCH_FAST=1:
- the check polls the log every 0.5 s until the step's log_regex (and files) show up after the reload, then waits
BENCH_FAST_GRACE (2 s) for errors that follow. Steps with nothing positive to wait for, min_errors or (by default) a
log_not_regex keep the full wait_seconds. Only the waiting changes: score judges the step as before;
- after a step, the reference files are written only when they differ, the reload is polled every 0.5 s, and the next
snapshot waits for BENCH_FAST_QUIET (1.5 s) of quiet log instead of 6 s;
- every tool line in the trace has its duration ("ms"), so the time a write spends waiting for its reload is visible.
Measured with qwen3.6:35b-mlx and the r11 MCP jar (r13, Oct 14 afternoon):
- reference pass 71/71 in 13 min (30 with fixed sleeps);
- three runs 67, 63 and 64 of 71 in 45, 55 and 48 min, against 65, 68 and 66 in 54-57 min with fixed sleeps;
- time outside the model went from about 33 to about 20 min per run.
The aggressive settings (grace 1 s, no full wait for log_not_regex) saved nothing (46 min). Known: a shorter quiet lets
an error of the step before land in the next step's count (one case in three runs); counting only the errors after the
step began is the next change.
Co-Authored-By: Claude Opus 5.5 (1M context) <no...@anthropic.com>
Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m
---
ai-benchmark/agent_mcp_stepwise.py | 118 ++++++++++++++++++++++++++++++++-----
1 file changed, 103 insertions(+), 15 deletions(-)
diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py
index 22237ba..0c40c46 100755
--- a/ai-benchmark/agent_mcp_stepwise.py
+++ b/ai-benchmark/agent_mcp_stepwise.py
@@ -154,6 +154,13 @@ def error_key(l):
INFRA_TIMEOUT = int(os.environ.get("BENCH_INFRA_TIMEOUT", "300")) # seconds to wait for `camel infra` services
+# BENCH_FAST=1: wait for what a step checks instead of fixed sleeps (Oct 14 2026: 22 of a run's 55 minutes were
+# fixed sleeps, while dev mode now reloads within ~2 s of a write, CAMEL-25032)
+FAST = os.environ.get("BENCH_FAST") == "1"
+FAST_POLL = float(os.environ.get("BENCH_FAST_POLL", "0.5")) # seconds between polls of the log
+FAST_GRACE = float(os.environ.get("BENCH_FAST_GRACE", "2")) # after the checks match: time for errors that follow
+FAST_QUIET = float(os.environ.get("BENCH_FAST_QUIET", "1.5")) # after a reference write: the log quiet this long
+FAST_NEG_FULL = os.environ.get("BENCH_FAST_NEG_FULL", "1") == "1" # a step that forbids output waits the full time
def infra_start(services):
@@ -219,6 +226,69 @@ def error_snapshot(mcp, name):
return records, count, reloads, before_msgs
+def project_file(project, fname):
+ """A file of the project by path, else by name anywhere in it (as score reads the files of a check)."""
+ content = read(os.path.join(project, fname))
+ if not content:
+ for root, dirs, files in os.walk(project):
+ dirs[:] = [d for d in dirs if not d.startswith(".")]
+ if os.path.basename(fname) in files:
+ return read(os.path.join(root, os.path.basename(fname)))
+ return content
+
+
+def wait_for_check(mcp, name, chk, cfg, project, before, reload_at):
+ """BENCH_FAST: instead of sleeping wait_seconds after the reload, poll until what the step checks is there (the
+ log lines after the reload, the files the app writes), then FAST_GRACE seconds for errors that follow. The full
+ wait_seconds when the step has nothing positive to wait for, expects errors, or (FAST_NEG_FULL) forbids output.
+ Only how long to wait changes: score judges the step afterwards exactly as before. Returns the seconds waited."""
+ t0, limit = time.time(), cfg["wait_seconds"]
+ regexes = chk.get("log_regex")
+ regexes = [] if regexes is None else (regexes if isinstance(regexes, list) else [regexes])
+ files = chk.get("files") or {}
+ if (not regexes and not files) or chk.get("min_errors") or (FAST_NEG_FULL and chk.get("log_not_regex")):
+ time.sleep(limit)
+ return limit
+ seen_msgs = before.get("log_records") or set()
+ need = chk.get("min_log", 1)
+
+ def msg(l): return (l.get("message") or l.get("msg") or "") + ("\n" + l["detail"] if l.get("detail") else "")
+
+ def fresh_ok(l):
+ if not isinstance(l, dict) or (l.get("level") or "").upper() not in ("INFO", "WARN"):
+ return False
+ if any(t in msg(l) for t in ("Error reloading routes", "Reload failed", "Failed to create route")):
+ return False
+ if error_key(l) in seen_msgs:
+ return False
+ t = tod(l.get("time") or l.get("timestamp"))
+ return reload_at is None or t is None or t >= reload_at
+ while time.time() - t0 < limit:
+ fresh = [msg(l) for l in log_lines(mcp, name, LOG_WINDOW) if fresh_ok(l)]
+ logs_ok = all(sum(1 for m in fresh if re.search(rx, m, re.S)) >= need for rx in regexes)
+ files_ok = all(re.search(rx, project_file(project, f) or "") for f, rx in files.items())
+ if logs_ok and files_ok:
+ time.sleep(max(0.0, min(FAST_GRACE, limit - (time.time() - t0))))
+ return round(time.time() - t0, 1)
+ time.sleep(FAST_POLL)
+ return round(time.time() - t0, 1)
+
+
+def wait_quiet(mcp, name, quiet, limit):
+ """BENCH_FAST: wait until the log has had no new record for quiet seconds (at most limit): the burst of a reload is
+ over, so the next step's snapshot is taken in a quiet state. Returns the seconds waited."""
+ t0 = time.time(); last = None; since = t0
+ while time.time() - t0 < limit:
+ keys = sorted(error_key(l) for l in log_lines(mcp, name, 20) if isinstance(l, dict))
+ fp = tuple(keys[-3:])
+ if fp != last:
+ last, since = fp, time.time()
+ elif time.time() - since >= quiet:
+ break
+ time.sleep(FAST_POLL)
+ return round(time.time() - t0, 1)
+
+
def probe(pr):
"""One HTTP request of a step check: {"method", "url", "headers", "body", "expect_status", "body_regex"}; the
server may still be settling after the reload, so the request is retried a few times until the expected status."""
@@ -269,11 +339,12 @@ def score(step, project, cfg, mcp, name, before):
result["file_ok"] = False
if chk.get("props_regex") and not re.search(chk["props_regex"], props):
result["props_ok"] = False
- # round 2: on macOS the JDK WatchService polls about every 10 s, so a dev-mode reload lands up to 10 s after the
- # write; wait for a reload record newer than the step's snapshot (up to 25 s), then the example's own wait
+ # wait for a reload record newer than the step's snapshot (up to 24 s), then the example's own wait (round 2 waited
+ # for the JDK WatchService, which polled about every 10 s on macOS; since CAMEL-25032 dev mode scans every 2 s)
seen_reloads = before.get("reload_records") or set()
reload_at = None
- for _ in range(12):
+ polls, gap = (int(24 / FAST_POLL), FAST_POLL) if FAST else (12, 2)
+ for _ in range(polls):
fresh_reloads = [l for l in log_lines(mcp, name, LOG_WINDOW)
if isinstance(l, dict) and is_reload(l) and reload_key(l) not in seen_reloads]
if fresh_reloads:
@@ -283,12 +354,15 @@ def score(step, project, cfg, mcp, name, before):
times = [t for t in (tod(l.get("time") or l.get("timestamp")) for l in fresh_reloads) if t is not None]
reload_at = max(times) if times else None
break
- time.sleep(2)
+ time.sleep(gap)
# CAMEL-24886: probes are the README's curl: an HTTP request after the reload, its status and body checked
result["probes"] = [probe(pr) for pr in (chk.get("probes") or [])]
if any(not pr["ok"] for pr in result["probes"]):
result["log_ok"] = False
- time.sleep(cfg["wait_seconds"])
+ if FAST:
+ result["waited"] = wait_for_check(mcp, name, chk, cfg, project, before, reload_at)
+ else:
+ time.sleep(cfg["wait_seconds"])
# (after the wait: a file the app writes on the reload, an outbox or an archive, exists only then)
# round 2: other files of the project (a Java bean, beans.yaml, a stylesheet, a dropped data file)
for fname, rx in (chk.get("files") or {}).items():
@@ -507,17 +581,20 @@ def main():
args["nameOrPid"] = name
if fn["name"] in ("camel_write_file", "camel_edit_file"):
writes += 1
+ t_call = time.time()
try:
out = mcp.call(fn["name"], args)
except Exception as e:
out = "ERROR: " + str(e)
+ # how long the tool took (a write waits for the dev-mode reload: CAMEL-25661)
+ ms = round((time.time() - t_call) * 1000)
if fn["name"] in ("camel_write_file", "camel_edit_file") and ('"invalid"' in out
or '"not-found"' in out or '"ambiguous"' in out or out.startswith("ERROR")):
refused += 1
elif fn["name"] in ("camel_write_file", "camel_edit_file"):
before["last_write"] = error_snapshot(mcp, name)
out = out[:TOOL_RESULT_CAP]
- trace.write(json.dumps({"tool": fn["name"], "args": {k: (v if k != "content" else v[:1500]) for k, v in args.items()}, "result": out[:800]}) + "\n"); trace.flush()
+ trace.write(json.dumps({"tool": fn["name"], "ms": ms, "args": {k: (v if k != "content" else v[:1500]) for k, v in args.items()}, "result": out[:800]}) + "\n"); trace.flush()
messages.append({"role": "tool", "content": out, "tool_name": fn["name"]})
continue
answer = msg.get("content") or ""
@@ -551,15 +628,26 @@ def main():
other = os.path.join(root, os.path.basename(fname))
if os.path.basename(fname) in files and os.path.abspath(other) != os.path.abspath(os.path.join(project, fname)):
os.remove(other)
- apply_reference(project, cfg, step["reference"], mcp, name)
- # the checkpoint write reloads the app too (on macOS up to 10 s later): wait for that reload and let
- # its consumers settle, so the next step's error and log snapshot is taken in a quiet state
- seen = error_snapshot(mcp, name)[2]
- for _ in range(12):
- time.sleep(2)
- if any(reload_key(l) not in seen for l in log_lines(mcp, name, LOG_WINDOW) if isinstance(l, dict) and is_reload(l)):
- break
- time.sleep(6)
+ if FAST:
+ # files that already equal the reference are not written again: no reload, nothing to wait for
+ if any(read(os.path.join(project, f)) != c for f, c in step["reference"].items()):
+ seen = error_snapshot(mcp, name)[2]
+ apply_reference(project, cfg, step["reference"], mcp, name)
+ for _ in range(int(24 / FAST_POLL)):
+ time.sleep(FAST_POLL)
+ if any(reload_key(l) not in seen for l in log_lines(mcp, name, LOG_WINDOW) if isinstance(l, dict) and is_reload(l)):
+ break
+ res["settled"] = wait_quiet(mcp, name, FAST_QUIET, 6)
+ else:
+ apply_reference(project, cfg, step["reference"], mcp, name)
+ # the checkpoint write reloads the app too: wait for that reload and let its consumers settle, so
+ # the next step's error and log snapshot is taken in a quiet state
+ seen = error_snapshot(mcp, name)[2]
+ for _ in range(12):
+ time.sleep(2)
+ if any(reload_key(l) not in seen for l in log_lines(mcp, name, LOG_WINDOW) if isinstance(l, dict) and is_reload(l)):
+ break
+ time.sleep(6)
if not res["ok"]:
messages.append({"role": "user", "content": "I fixed that step myself; the files now contain the correct version. Continue with the next request."})
json.dump(results, open(os.path.join(OUT, "results.json"), "w"), indent=1)
(camel-performance-tests) 02/06: ai-benchmark: BENCH_FAST counts only the errors after a step began, BENCH_SERVER_INSTRUCTIONS, and data dirs for the ladder
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit 4fae7f9590ec10dd6e340f1af003135663b31245
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Wed Oct 14 21:49:49 2026 +0200
ai-benchmark: BENCH_FAST counts only the errors after a step began, BENCH_SERVER_INSTRUCTIONS, and data dirs for the ladder
- With BENCH_FAST, an error logged before the step began, or an error record with an earlier timestamp, belongs to the
step before, however late it came in. The shorter quiet after a reference write let one land in the next step's count
(r13, one case in three runs).
- BENCH_SERVER_INSTRUCTIONS=1 appends the instructions from the MCP server's initialize answer (CAMEL-25646) to the
system prompt, as agent hosts show them to the model.
- connect-file-processing, transform-csv-to-json and fail-well-error-handling declare the folders their routes write
into as camel.jbang.dataDir (CAMEL-25670). With --source-dir, dev mode reloaded connect-file-processing every 2 s for
its own files.
Co-Authored-By: Claude Opus 5.5 (1M context) <no...@anthropic.com>
Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m
---
ai-benchmark/agent_mcp_stepwise.py | 33 ++++++++++++++++++++++++++++++---
ai-benchmark/gen_stepwise.py | 9 ++++++---
2 files changed, 36 insertions(+), 6 deletions(-)
diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py
index 0c40c46..1fa104f 100755
--- a/ai-benchmark/agent_mcp_stepwise.py
+++ b/ai-benchmark/agent_mcp_stepwise.py
@@ -161,6 +161,9 @@ FAST_POLL = float(os.environ.get("BENCH_FAST_POLL", "0.5")) # seconds betwee
FAST_GRACE = float(os.environ.get("BENCH_FAST_GRACE", "2")) # after the checks match: time for errors that follow
FAST_QUIET = float(os.environ.get("BENCH_FAST_QUIET", "1.5")) # after a reference write: the log quiet this long
FAST_NEG_FULL = os.environ.get("BENCH_FAST_NEG_FULL", "1") == "1" # a step that forbids output waits the full time
+# BENCH_SERVER_INSTRUCTIONS=1: append the instructions the MCP server sends in its initialize answer (CAMEL-25646) to
+# the system prompt, as agent hosts such as Claude Code show them to the model
+SERVER_INSTRUCTIONS = os.environ.get("BENCH_SERVER_INSTRUCTIONS") == "1"
def infra_start(services):
@@ -422,11 +425,29 @@ def score(step, project, cfg, mcp, name, before):
result["log_ok"] = False
# round 2: errors are counted per step (the log window and the error file keep an earlier step's stack traces)
seen = before.get("error_records") or set()
- result["errors"] = sum(1 for l in lines if isinstance(l, dict) and lvl(l) == "ERROR" and error_key(l) not in seen)
+ began = before.get("began")
+
+ def in_step(l):
+ # BENCH_FAST: an error logged before the step began is the step before's, however late it came in
+ if not FAST or began is None:
+ return True
+ t = tod(l.get("time") or l.get("timestamp"))
+ b = tod(time.strftime("%H:%M:%S", time.localtime(began)))
+ if t is None or b is None:
+ return True
+ return t + (86400 if t < b - 43200 else 0) >= b - 0.5
+ result["errors"] = sum(1 for l in lines if isinstance(l, dict) and lvl(l) == "ERROR" and error_key(l) not in seen
+ and in_step(l))
data, _ = jcall(mcp, "camel_get_errors", {"name": name})
if isinstance(data, dict):
errs = data.get("errors", []) or []
- result["errors"] += max(0, len(errs) - before.get("error_count", 0))
+ if FAST and began is not None:
+ # the error records carry their time (epoch millis): only the ones at or after the step began
+ new = errs[before.get("error_count", 0):]
+ result["errors"] += sum(1 for e in new if not isinstance(e, dict) or not isinstance(e.get("timestamp"), (int, float))
+ or e["timestamp"] >= (began - 0.5) * 1000)
+ else:
+ result["errors"] += max(0, len(errs) - before.get("error_count", 0))
# what they were, not just how many: a step that fails on errors alone cannot be judged from a count
result["error_detail"] = [str(e)[:300] for e in errs[before.get("error_count", 0):]][:5]
diff = list(difflib.unified_diff(before["route"].splitlines(), route.splitlines(), lineterm="", n=0))
@@ -465,7 +486,8 @@ def main():
project = os.path.join(HERE, project)
os.makedirs(OUT, exist_ok=True)
log = open(os.path.join(OUT, "run.log"), "a")
- mcp = McpClient(MCP_URL); mcp.initialize(); all_tools = mcp.list_tools()
+ mcp = McpClient(MCP_URL); init = mcp.initialize(); all_tools = mcp.list_tools()
+ instructions = ((init or {}).get("result") or {}).get("instructions") if isinstance(init, dict) else None
tools = to_ollama_tools(all_tools, BARE_TOOLS if BARE else SHARED + EXTRA)
print(f"tools: {[t['function']['name'] for t in tools]}", file=log, flush=True)
@@ -517,6 +539,10 @@ def main():
time.sleep(6)
base_system = SYSTEM.replace("{directory}", project).replace("{name}", name)
+ if SERVER_INSTRUCTIONS:
+ if instructions:
+ base_system += "\nThe Camel MCP server's own instructions:\n" + instructions.strip() + "\n"
+ print(f"server instructions: {len(instructions or '')} chars", file=log, flush=True)
if BARE:
# the guidelines about tools it does not have would only confuse it
base_system = "\n".join(l for l in base_system.splitlines()
@@ -541,6 +567,7 @@ def main():
before = {"route": read(os.path.join(project, cfg["route_file"])), "props": read(os.path.join(project, cfg.get("props_file", "application.properties")))}
before["error_records"], before["error_count"], before["reload_records"], before["log_records"] \
= error_snapshot(mcp, name)
+ before["began"] = time.time()
trace = open(os.path.join(OUT, f"step{sid}.trace.jsonl"), "w")
messages.append({"role": "user", "content": step["request"]})
calls = 0; tokens = 0; t0 = time.time(); writes = 0; refused = 0; answer = ""
diff --git a/ai-benchmark/gen_stepwise.py b/ai-benchmark/gen_stepwise.py
index 66d97ce..4b5eb65 100644
--- a/ai-benchmark/gen_stepwise.py
+++ b/ai-benchmark/gen_stepwise.py
@@ -14,6 +14,9 @@ import datetime, json, os, shutil, sys
HERE = os.path.dirname(os.path.abspath(__file__))
STEPS_DIR = os.path.join(HERE, "steps-ladder")
+# CAMEL-25670: dev mode with --source-dir reloads the routes for every file they write into the project's own folders
+# (connect-file-processing reloaded every 2 s, r13); camel.jbang.dataDir leaves those folders out (Camel 4.24)
+DATA_DIR = "camel.jbang.dataDir="
PROJECTS = os.path.join(HERE, "stepwise-ladder")
SEEDS = os.path.join(HERE, "seed")
@@ -396,7 +399,7 @@ csv_s5 = route("csv-to-json", FILE_INBOX_CSV, UNMARSHAL_CSV + split_body(CSV_HDR
EXAMPLES.append({
"name": "transform-csv-to-json", "route_file": CSV, "props_file": "application.properties", "wait_seconds": 8,
"seed": "transform-csv-to-json",
- "initial": {CSV: csv_s1, "application.properties": ""},
+ "initial": {CSV: csv_s1, "application.properties": DATA_DIR + "inbox,outbox\n"},
"steps": [
{"request": "Unmarshal the CSV with the csv data format so the body becomes a list of maps keyed by the header row (useMaps and captureHeaderRecord), and log \"Invoices as a list of maps: ${body}\".",
"check": {"file_regex": "csv", "log_regex": "Invoices as a list of maps: \\[\\{invoiceId=INV-2001"},
@@ -542,7 +545,7 @@ eh_s4 = DLC + ON_DECLINED + CHECKOUT + "\n" + provider(FLAKY + DECLINED, True) +
EXAMPLES.append({
"name": "fail-well-error-handling", "route_file": EH, "props_file": "application.properties", "wait_seconds": 12,
"seed": "fail-well-error-handling",
- "initial": {EH: eh_s1, "application.properties": ""},
+ "initial": {EH: eh_s1, "application.properties": DATA_DIR + "orders,parked\n"},
"steps": [
{"request": "Add a top-level errorHandler with a deadLetterChannel to direct:parked (maximumRedeliveries 2, redeliveryDelay 1000, retryAttemptedLogLevel WARN) and a route parked from direct:parked that logs \"Order ${body[orderId]} parked for manual review: ${exception.message}\", so the declined order is parked after the retries instead of dropped with a stack trace.",
"check": {"file_regex": "deadLetterChannel", "log_regex": ["Order ORD-1003 parked for manual review: card declined", "Payment for ORD-1001 charged, 2 line\\(s\\)"], "errors_ok": True},
@@ -578,7 +581,7 @@ fp_s5 = ON_INVALID + COURIER + "\n" + route("invoices", INBOX_JSON_FAILED, UNMAR
EXAMPLES.append({
"name": "connect-file-processing", "route_file": FP, "props_file": "application.properties", "wait_seconds": 8,
"seed": "connect-file-processing",
- "initial": {FP: fp_s1, "application.properties": ""},
+ "initial": {FP: fp_s1, "application.properties": DATA_DIR + "samples,inbox,archive\n"},
"steps": [
{"request": "The invoices route must only take the JSON files: include .*\\.json, and move each processed file to the done directory instead of the default .camel directory. The driver's note note-2005.txt must stay untouched in inbox.",
"check": {"file_regex": "move: done", "file_regex2": "include", "log_regex": "Received invoice-2004.json", "log_not_regex": "Received note-2005", "files": {"inbox/note-2005.txt": "Handwritten note", "done/invoice-2001.json": "INV-2001"}},
(camel-performance-tests) 04/06: ai-benchmark: the server restart stops only its own server, and the postgres wait reads the new JSON errors
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit f262f3e8ae396bbc15458a2c8b117798d845e905
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Fri Oct 16 22:48:57 2026 +0200
ai-benchmark: the server restart stops only its own server, and the postgres wait reads the new JSON errors
start-server.sh stopped every camel-jbang-mcp runner jar on the machine, and Claude Code sessions run such jars as
stdio MCP servers of their own; it now stops only the server on its port. It also stops the tail that feeds the
server: each restart left one behind (26 since Oct 13).
Since CAMEL-25392 a failing `camel infra get <svc> --json` prints a JSON error ({"status":"error","code":3,...})
while the service starts, and the wait took any JSON for the service being up. It now waits for an answer that is
not an error.
Co-Authored-By: Claude Opus 5.5 (1M context) <no...@anthropic.com>
Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m
---
ai-benchmark/agent_local.py | 5 ++++-
ai-benchmark/agent_mcp_stepwise.py | 7 +++++--
ai-benchmark/start-server.sh | 7 ++++++-
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/ai-benchmark/agent_local.py b/ai-benchmark/agent_local.py
index 9de7e9a..2d3de9d 100755
--- a/ai-benchmark/agent_local.py
+++ b/ai-benchmark/agent_local.py
@@ -182,7 +182,10 @@ def infra_start(services):
m = re.search(r"\{.*\}", out, re.S)
if m:
try:
- data[svc] = json.loads(m.group(0)); break
+ d = json.loads(m.group(0))
+ # since CAMEL-25392 an error is JSON too: {"status":"error","code":3,...} until the service is up
+ if d.get("status") != "error":
+ data[svc] = d; break
except json.JSONDecodeError:
pass
time.sleep(5)
diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py
index 1548bf1..70b2d0f 100755
--- a/ai-benchmark/agent_mcp_stepwise.py
+++ b/ai-benchmark/agent_mcp_stepwise.py
@@ -188,8 +188,11 @@ def infra_start(services):
m = re.search(r"\{.*\}", out, re.S)
if m:
try:
- data[svc] = json.loads(m.group(0))
- break
+ d = json.loads(m.group(0))
+ # since CAMEL-25392 an error is JSON too: {"status":"error","code":3,...} until the service is up
+ if d.get("status") != "error":
+ data[svc] = d
+ break
except json.JSONDecodeError:
pass
time.sleep(5)
diff --git a/ai-benchmark/start-server.sh b/ai-benchmark/start-server.sh
index 9bc15de..2bbe65d 100755
--- a/ai-benchmark/start-server.sh
+++ b/ai-benchmark/start-server.sh
@@ -3,7 +3,12 @@
# Set CAMEL_MCP_JAR to run a locally built camel-jbang-mcp runner jar instead of the CLI plugin.
cd "$(dirname "$0")"
PORT="${MCP_PORT:-9090}"
-pkill -f "camel-jbang-mcp.*runner.jar" 2>/dev/null; pkill -f "camel mcp --http" 2>/dev/null; sleep 1
+# only the server on this port: Claude Code sessions run camel-jbang-mcp runner jars of their own (stdio MCP servers).
+# The wrapper's tail goes too, or every restart leaves an orphaned tail behind
+for w in $(pgrep -f "^sh -c tail -f /dev/null \| .*(quarkus.http.port=$PORT |camel mcp --http --port=$PORT )"); do
+ pkill -P $w 2>/dev/null; kill $w 2>/dev/null
+done
+pkill -f "quarkus.http.port=$PORT .*camel-jbang-mcp" 2>/dev/null; pkill -f "camel mcp --http --port=$PORT" 2>/dev/null; sleep 1
if [[ -n "${CAMEL_MCP_JAR:-}" ]]; then
nohup sh -c "tail -f /dev/null | java -Dquarkus.http.host-enabled=true -Dquarkus.http.host=0.0.0.0 -Dquarkus.http.port=$PORT -Dquarkus.log.level=WARN -jar $CAMEL_MCP_JAR" > mcp-server.log 2>&1 &
else
(camel-performance-tests) 03/06: ai-benchmark: 25 tool calls per step instead of 20
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-performance-tests.git
commit d889c9188c02a39c55984c22f12e432183c87fcf
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Thu Oct 15 07:25:22 2026 +0200
ai-benchmark: 25 tool calls per step instead of 20
Of the 78 steps that hit 20 calls in r12-r14, 24 passed anyway and 30 still had an accepted write in their last 5
calls, so they were still making progress. The steps that pass need few calls (median 4, p95 13, p99 18), so a higher
cap mostly gives the ones that were still writing room to finish. Runs before Oct 15 2026 had 20.
Co-Authored-By: Claude Opus 5.5 (1M context) <no...@anthropic.com>
Claude-Session: https://claude.ai/code/session_01STT6whBgK1AqsSsUKrnE8m
---
ai-benchmark/agent_mcp_stepwise.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/ai-benchmark/agent_mcp_stepwise.py b/ai-benchmark/agent_mcp_stepwise.py
index 1fa104f..1548bf1 100755
--- a/ai-benchmark/agent_mcp_stepwise.py
+++ b/ai-benchmark/agent_mcp_stepwise.py
@@ -11,7 +11,7 @@ errors via camel_get_errors, diff size, reference checkpoint after each step.
Env: BENCH_MODEL (ollama model), MCP_URL (default http://127.0.0.1:9090/mcp), BENCH_TAG (results folder name),
BENCH_STEPS (the steps file, default steps.json),
-BENCH_TOOL_CALLS (max tool calls per step, default 12).
+BENCH_TOOL_CALLS (max tool calls per step, default 25; 20 until Oct 15 2026).
"""
import difflib, json, os, re, subprocess, sys, time, urllib.error, urllib.request
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
@@ -23,7 +23,8 @@ MCP_URL = os.environ.get("MCP_URL", "http://127.0.0.1:9090/mcp")
HERE = os.path.dirname(os.path.abspath(__file__))
TAG = os.environ.get("BENCH_TAG", "mcp-" + MODEL.replace(":", "_").replace("/", "_"))
OUT = os.path.join(HERE, "stepwise", TAG)
-MAX_TOOL_CALLS = int(os.environ.get("BENCH_TOOL_CALLS", "20"))
+# 25 since Oct 15 2026 (20 before, r12-r14): of the steps that hit 20, 38% were still writing in their last 5 calls
+MAX_TOOL_CALLS = int(os.environ.get("BENCH_TOOL_CALLS", "25"))
# the conversation grows over an example's steps (logs with stack traces are the bulk); at 32k a long example ran out
# of context and the model stopped after a few words without a tool call (round 3, circuit-breaker steps 2-3)
NUM_CTX = int(os.environ.get("BENCH_NUM_CTX", "65536"))