You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@juneau.apache.org by ja...@apache.org on 2026/05/27 13:26:36 UTC
(juneau) branch master updated: build: add CI perf-guard with --enforce-perf flag, per-baseline tolerance band, and Surefire XML bucketed assertions (TODO-139)
This is an automated email from the ASF dual-hosted git repository.
jamesbognar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/juneau.git
The following commit(s) were added to refs/heads/master by this push:
new b32d9ae831 build: add CI perf-guard with --enforce-perf flag, per-baseline tolerance band, and Surefire XML bucketed assertions (TODO-139)
b32d9ae831 is described below
commit b32d9ae83149bb13795507f2548db26130c07c97
Author: James Bognar <ja...@apache.org>
AuthorDate: Wed May 27 09:26:31 2026 -0400
build: add CI perf-guard with --enforce-perf flag, per-baseline tolerance band, and Surefire XML bucketed assertions (TODO-139)
---
Jenkinsfile | 4 +
juneau-utest/perf-baseline.txt | 25 +++++
pom.xml | 3 +
scripts/ci-perf-guard.py | 223 +++++++++++++++++++++++++++++++++++++++++
scripts/test.py | 129 +++++++++++++++++++++++-
5 files changed, 383 insertions(+), 1 deletion(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index e348805452..840c67ca51 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -35,6 +35,10 @@ timestamps {
}
}
+ stage ('Juneau-Java-17 - Perf guard') {
+ sh 'python3 scripts/ci-perf-guard.py || true'
+ }
+
stage ('Juneau-Java-17 - Post build actions') {
step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: 'dev@juneau.apache.org', sendToIndividuals: true])
}
diff --git a/juneau-utest/perf-baseline.txt b/juneau-utest/perf-baseline.txt
new file mode 100644
index 0000000000..e175a7397f
--- /dev/null
+++ b/juneau-utest/perf-baseline.txt
@@ -0,0 +1,25 @@
+# juneau-utest performance baselines.
+#
+# Line format: <seconds> # <label>; as of <YYYY-MM-DD>; tolerance ±<pct>%; set by FINISHED-<id>.
+#
+# Line 1 — suite wall-clock (subprocess time of `mvn test`).
+# Used by: scripts/test.py --enforce-perf
+# Measured on: developer M1 Pro laptop (warm cache).
+#
+# Lines 2–4 — per-bucket Surefire XML aggregate totals.
+# Used by: scripts/ci-perf-guard.py (reads Surefire XML or --timing-log JSONL)
+# Measured on: developer M1 Pro laptop via ~/.cache/juneau-push-timings/ JSONL.
+#
+# To re-baseline: run `python3 scripts/test.py --full --timing-log /tmp/juneau-timing.jsonl`,
+# read the wall-clock and bucket values, update this file AND add a ## Perf block to the
+# FINISHED archive in the same PR. Full procedure in todo/PERF-BASELINE.md.
+#
+# NOTE: The per-bucket baselines (lines 2–4) were measured on a developer M1 Pro and will
+# differ from CI Linux values. The first Jenkins run after this guard lands will likely
+# fail; follow the first-CI-run calibration procedure in todo/PERF-BASELINE.md to set
+# the canonical CI baseline.
+
+99 # suite wall-clock; as of 2026-05-27; tolerance ±20%; set by FINISHED-95.
+60 # core; as of 2026-05-27; tolerance ±20%; set by FINISHED-139.
+3 # container.springboot; as of 2026-05-27; tolerance ±20%; set by FINISHED-139.
+1 # container.jetty; as of 2026-05-27; tolerance ±20%; set by FINISHED-139.
diff --git a/pom.xml b/pom.xml
index b8fb3e157d..2ce127e27a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -165,6 +165,9 @@
<!-- ServiceLoader provider files - plain text, no comment syntax for license headers -->
<exclude>**/META-INF/services/**</exclude>
+
+ <!-- Performance baseline data file - plain text metrics, no license header syntax -->
+ <exclude>**/perf-baseline.txt</exclude>
</excludes>
<consoleOutput>true</consoleOutput>
<!-- Note: useEclipseDefaultExcludes does not work for subprojects. See RAT-107 -->
diff --git a/scripts/ci-perf-guard.py b/scripts/ci-perf-guard.py
new file mode 100644
index 0000000000..09f524dd49
--- /dev/null
+++ b/scripts/ci-perf-guard.py
@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+# ***************************************************************************************************************************
+# * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations under the License.
+# ***************************************************************************************************************************
+"""
+CI perf guard for Apache Juneau.
+
+Reads per-bucket Surefire XML totals (or a --timing-log JSONL written by scripts/test.py)
+and compares them against the baselines in juneau-utest/perf-baseline.txt. Intended to run
+in Jenkinsfile's post-build block after the main Maven build has already produced Surefire
+XML reports.
+
+Usage:
+ python3 scripts/ci-perf-guard.py [--timing-log <path>] [--dry-run]
+
+Options:
+ --timing-log <path> Read actual times from a JSONL written by scripts/test.py --timing-log.
+ When omitted, reads Surefire XML directly from
+ juneau-utest/target/surefire-reports/{core,container}/.
+ --dry-run Print detected totals and baselines but do not exit non-zero on breach.
+ --help Show this help message.
+
+Env vars:
+ JUNEAU_CI_PERF_THRESHOLD Tolerance as a decimal fraction (default 0.20 = ±20%).
+ Separate from JUNEAU_PUSH_TIMING_THRESHOLD.
+
+Baseline lines in perf-baseline.txt (lines 2–4, Surefire XML aggregate values):
+ line 2 — core bucket total
+ line 3 — container.springboot bucket total
+ line 4 — container.jetty bucket total
+
+NOTE: The per-bucket baselines were calibrated on a developer M1 Pro laptop. The first
+Jenkins run after this guard is introduced will likely show different values; follow the
+first-CI-run calibration procedure in todo/PERF-BASELINE.md to set the canonical CI baseline.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+
+def parse_surefire_dir(directory: Path) -> float:
+ """Return the sum of Surefire XML 'time' attributes under directory."""
+ total = 0.0
+ if not directory.exists():
+ return total
+ for xml_file in sorted(directory.glob("TEST-*.xml")):
+ try:
+ root = ET.parse(xml_file).getroot()
+ total += float(root.attrib.get("time", 0.0))
+ except Exception:
+ continue
+ return total
+
+
+def container_bucket(module_dir: Path, class_name: str) -> str:
+ """Classify a container test class as container.springboot or container.jetty."""
+ source = module_dir / "src" / "test" / "java" / Path("/".join(class_name.split("."))).with_suffix(".java")
+ if source.exists():
+ try:
+ content = source.read_text(encoding="utf-8")
+ if "SpringbootTest" in content:
+ return "container.springboot"
+ if "JettyMicroserviceTest" in content:
+ return "container.jetty"
+ except OSError:
+ pass
+ return "container.springboot" if "springboot" in class_name.lower() else "container.jetty"
+
+
+def parse_container_subbuckets(module_dir: Path, directory: Path) -> dict:
+ """Return {'container.springboot': float, 'container.jetty': float} Surefire XML totals."""
+ buckets: dict[str, float] = {"container.springboot": 0.0, "container.jetty": 0.0}
+ if not directory.exists():
+ return buckets
+ for xml_file in sorted(directory.glob("TEST-*.xml")):
+ try:
+ root = ET.parse(xml_file).getroot()
+ bucket = container_bucket(module_dir, root.attrib.get("name", ""))
+ buckets[bucket] += float(root.attrib.get("time", 0.0))
+ except Exception:
+ continue
+ return buckets
+
+
+def read_baselines_from_file(baseline_file: Path) -> dict:
+ """Parse perf-baseline.txt; positions 2–4 (0-based indices 1–3) are the per-bucket baselines."""
+ keys = ["suite", "core", "container.springboot", "container.jetty"]
+ result: dict[str, float] = {}
+ if not baseline_file.exists():
+ return result
+ idx = 0
+ for line in baseline_file.read_text(encoding="utf-8").splitlines():
+ stripped = line.strip()
+ if not stripped or stripped.startswith("#"):
+ continue
+ if idx >= len(keys):
+ break
+ try:
+ result[keys[idx]] = float(stripped.split()[0])
+ except (ValueError, IndexError):
+ pass
+ idx += 1
+ return result
+
+
+def read_actuals_from_jsonl(log_path: Path) -> dict:
+ """Return {execution: wallclock_s} for the latest run_id in the JSONL."""
+ rows: list[dict] = []
+ for line in log_path.read_text(encoding="utf-8").splitlines():
+ stripped = line.strip()
+ if not stripped:
+ continue
+ try:
+ rows.append(json.loads(stripped))
+ except json.JSONDecodeError:
+ continue
+ if not rows:
+ return {}
+ latest_run_id = rows[-1].get("run_id")
+ result: dict[str, float] = {}
+ for row in rows:
+ if row.get("run_id") == latest_run_id:
+ execution = row.get("execution", "")
+ result[execution] = float(row.get("wallclock_s", 0.0))
+ return result
+
+
+def read_actuals_from_surefire(repo_root: Path) -> dict:
+ """Read per-bucket Surefire XML aggregate times directly from target/surefire-reports/."""
+ module_dir = repo_root / "juneau-utest"
+ reports_root = module_dir / "target" / "surefire-reports"
+ core = parse_surefire_dir(reports_root / "core")
+ sub = parse_container_subbuckets(module_dir, reports_root / "container")
+ return {
+ "core": core,
+ "container.springboot": sub["container.springboot"],
+ "container.jetty": sub["container.jetty"],
+ }
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(add_help=False)
+ parser.add_argument("--timing-log")
+ parser.add_argument("--dry-run", action="store_true")
+ parser.add_argument("--help", "-h", action="store_true")
+ args, unknown = parser.parse_known_args()
+ if args.help:
+ print(__doc__)
+ return 0
+ if unknown:
+ print(f"Unknown option(s): {' '.join(unknown)}")
+ print(__doc__)
+ return 1
+
+ repo_root = Path(__file__).parent.parent
+ baseline_file = repo_root / "juneau-utest" / "perf-baseline.txt"
+ baselines = read_baselines_from_file(baseline_file)
+ if not baselines:
+ print(f"PERF-GUARD: baseline file not found or empty ({baseline_file}); skipping.")
+ return 0
+
+ if args.timing_log:
+ log_path = Path(args.timing_log).expanduser()
+ if not log_path.exists():
+ print(f"PERF-GUARD: timing log not found ({log_path}); falling back to Surefire XML.")
+ actuals = read_actuals_from_surefire(repo_root)
+ else:
+ actuals = read_actuals_from_jsonl(log_path)
+ else:
+ actuals = read_actuals_from_surefire(repo_root)
+
+ tolerance = float(os.environ.get("JUNEAU_CI_PERF_THRESHOLD", "0.20"))
+ pct = f"±{tolerance * 100:.0f}%"
+ guard_failed = False
+
+ total_actual = actuals.get("core", 0.0) + actuals.get("container.springboot", 0.0) + actuals.get("container.jetty", 0.0)
+ print(f"PERF-GUARD CI: detected total {total_actual:.1f}s (core {actuals.get('core', 0.0):.1f}s + springboot {actuals.get('container.springboot', 0.0):.1f}s + jetty {actuals.get('container.jetty', 0.0):.1f}s)")
+
+ for bucket in ("core", "container.springboot", "container.jetty"):
+ actual = actuals.get(bucket)
+ baseline = baselines.get(bucket)
+ if actual is None:
+ print(f"PERF-GUARD [{bucket}]: no data; skipping.")
+ continue
+ if baseline is None:
+ print(f"PERF-GUARD [{bucket}]: no baseline configured; skipping.")
+ continue
+ threshold = baseline * (1 + tolerance)
+ if actual > threshold:
+ print(
+ f"PERF-GUARD FAIL [{bucket}]: {actual:.1f}s "
+ f"(baseline {baseline}s, tolerance {pct}, threshold {threshold:.1f}s).\n"
+ f" Follow the re-baseline procedure in todo/PERF-BASELINE.md if this is intentional.\n"
+ f" NOTE: If this is the first CI run, the developer-machine baseline needs CI calibration."
+ )
+ guard_failed = True
+ else:
+ print(f"PERF-GUARD OK [{bucket}]: {actual:.1f}s (baseline {baseline}s, tolerance {pct}).")
+
+ if guard_failed and args.dry_run:
+ print("PERF-GUARD: dry-run mode; breach detected but not failing the build.")
+ return 0
+
+ return 1 if guard_failed else 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/test.py b/scripts/test.py
index 2aaf9b5294..6375a8f444 100755
--- a/scripts/test.py
+++ b/scripts/test.py
@@ -24,15 +24,29 @@ Options:
--verbose, -v Show full Maven output
--no-container Exclude @Tag("container") tests
--timing-log <path> Append per-bucket timing JSONL records
+ --enforce-perf Hard-fail if wall-clock exceeds perf-baseline.txt ±20% tolerance
--profile <module> Run one-shot JFR profile for module tests
--help, -h Show this help message
+
+Perf guard:
+ --enforce-perf compares the actual tests-only wall-clock against the suite baseline in
+ juneau-utest/perf-baseline.txt (line 1). Uses env-var JUNEAU_CI_PERF_THRESHOLD (default
+ 0.20, i.e. ±20%) — separate from JUNEAU_PUSH_TIMING_THRESHOLD.
+
+ When --timing-log is also passed the per-bucket Surefire XML totals are compared against
+ the core / container.springboot / container.jetty baselines (lines 2–4 of perf-baseline.txt).
+
+ Without --enforce-perf the check runs in warn-only mode (prints results, never exits non-zero
+ for a perf breach).
"""
import argparse
import json
+import os
import re
import subprocess
import sys
+import time
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from pathlib import Path
@@ -154,6 +168,111 @@ def write_timing_log(path: Path, passed: bool):
print(f"🕒 Timing metrics appended to {path}")
+def read_baselines(baseline_file: Path) -> dict:
+ """Read baselines from perf-baseline.txt.
+
+ Returns a dict mapping label to float:
+ 'suite' — line 1, suite wall-clock (used by --enforce-perf).
+ 'core' — line 2, core Surefire XML total.
+ 'container.springboot'— line 3, springboot Surefire XML total.
+ 'container.jetty' — line 4, jetty Surefire XML total.
+ """
+ keys = ["suite", "core", "container.springboot", "container.jetty"]
+ result = {}
+ if not baseline_file.exists():
+ return result
+ idx = 0
+ for line in baseline_file.read_text(encoding="utf-8").splitlines():
+ stripped = line.strip()
+ if not stripped or stripped.startswith("#"):
+ continue
+ if idx >= len(keys):
+ break
+ try:
+ value = float(stripped.split()[0])
+ result[keys[idx]] = value
+ idx += 1
+ except (ValueError, IndexError):
+ continue
+ return result
+
+
+def _read_latest_jsonl_run(log_path: Path) -> dict:
+ """Return {execution: wallclock_s} for the latest run_id in the JSONL."""
+ rows = []
+ for line in log_path.read_text(encoding="utf-8").splitlines():
+ stripped = line.strip()
+ if not stripped:
+ continue
+ try:
+ rows.append(json.loads(stripped))
+ except json.JSONDecodeError:
+ continue
+ if not rows:
+ return {}
+ latest_run_id = rows[-1].get("run_id")
+ result = {}
+ for row in rows:
+ if row.get("run_id") == latest_run_id:
+ execution = row.get("execution", "")
+ result[execution] = float(row.get("wallclock_s", 0.0))
+ return result
+
+
+def run_perf_guard(test_elapsed: float, baseline_file: Path, timing_log_path, enforce: bool) -> int:
+ """Run the v1 (suite wall-clock) and v2 (per-bucket Surefire XML) perf guards.
+
+ Returns exit code: 0 = pass or warn-only mode, 1 = threshold breached with enforce=True.
+ """
+ tolerance = float(os.environ.get("JUNEAU_CI_PERF_THRESHOLD", "0.20"))
+ baselines = read_baselines(baseline_file)
+ if not baselines:
+ print(f"PERF-GUARD: baseline file not found ({baseline_file}); skipping check.")
+ return 0
+
+ pct = f"±{tolerance * 100:.0f}%"
+ perf_failed = False
+
+ suite_baseline = baselines.get("suite")
+ if suite_baseline is not None:
+ threshold = suite_baseline * (1 + tolerance)
+ if test_elapsed > threshold:
+ print(
+ f"PERF-GUARD FAIL: tests took {test_elapsed:.1f}s "
+ f"(baseline {suite_baseline}s, tolerance {pct}, threshold {threshold:.1f}s).\n"
+ f"If this is intentional, bump juneau-utest/perf-baseline.txt and the FINISHED archive's ## Perf block in the same PR."
+ )
+ perf_failed = True
+ else:
+ print(f"PERF-GUARD OK: tests took {test_elapsed:.1f}s (baseline {suite_baseline}s, tolerance {pct}).")
+
+ if timing_log_path is not None:
+ log_path = Path(timing_log_path).expanduser()
+ if log_path.exists():
+ latest_run = _read_latest_jsonl_run(log_path)
+ for bucket in ("core", "container.springboot", "container.jetty"):
+ bucket_actual = latest_run.get(bucket)
+ bucket_baseline = baselines.get(bucket)
+ if bucket_actual is None or bucket_baseline is None:
+ continue
+ bucket_threshold = bucket_baseline * (1 + tolerance)
+ if bucket_actual > bucket_threshold:
+ print(
+ f"PERF-GUARD FAIL [{bucket}]: {bucket_actual:.1f}s "
+ f"(baseline {bucket_baseline}s, tolerance {pct}, threshold {bucket_threshold:.1f}s).\n"
+ f"If this is intentional, bump juneau-utest/perf-baseline.txt and the FINISHED archive's ## Perf block in the same PR."
+ )
+ perf_failed = True
+ else:
+ print(f"PERF-GUARD OK [{bucket}]: {bucket_actual:.1f}s (baseline {bucket_baseline}s, tolerance {pct}).")
+
+ if perf_failed:
+ if not enforce:
+ print("PERF-GUARD: warn-only mode (pass --enforce-perf to hard-fail on breach).")
+ return 1 if enforce else 0
+ return 0
+
+
def build(verbose=False):
return run_command("mvn clean install -DskipTests", verbose)
@@ -187,6 +306,7 @@ def main(): # NOSONAR python:S3776 -- Cognitive complexity is acceptable for th
parser.add_argument("--verbose", "-v", action="store_true")
parser.add_argument("--no-container", action="store_true")
parser.add_argument("--timing-log")
+ parser.add_argument("--enforce-perf", action="store_true")
parser.add_argument("--profile")
parser.add_argument("--help", "-h", action="store_true")
args, unknown = parser.parse_known_args()
@@ -223,7 +343,11 @@ def main(): # NOSONAR python:S3776 -- Cognitive complexity is acceptable for th
if test_only or full:
if full:
print("\n" + "=" * 80)
+ if not args.enforce_perf:
+ print("PERF-GUARD: warn-only mode (pass --enforce-perf to hard-fail on breach).")
+ test_start = time.time()
exit_code, last_test_output = test(verbose, no_container=args.no_container)
+ test_elapsed = time.time() - test_start
if exit_code != 0:
_, failures, errors = parse_test_results(last_test_output)
if failures is not None and errors is not None:
@@ -236,8 +360,11 @@ def main(): # NOSONAR python:S3776 -- Cognitive complexity is acceptable for th
write_timing_log(Path(args.timing_log), passed=(exit_code == 0))
if exit_code != 0:
return exit_code
+ baseline_file = Path(__file__).parent.parent / "juneau-utest" / "perf-baseline.txt"
+ perf_exit = run_perf_guard(test_elapsed, baseline_file, args.timing_log, enforce=args.enforce_perf)
+ if perf_exit != 0:
+ return perf_exit
return exit_code
if __name__ == '__main__':
sys.exit(main())
-