You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@groovy.apache.org by pa...@apache.org on 2026/07/05 00:01:42 UTC
(groovy) branch master updated: GROOVY-12127: Improve benchmark framework: Runner calibration/Deterministic dispatch metrics/Cold instance-dispatch bench
This is an automated email from the ASF dual-hosted git repository.
paulk-asert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/groovy.git
The following commit(s) were added to refs/heads/master by this push:
new 128c2780b8 GROOVY-12127: Improve benchmark framework: Runner calibration/Deterministic dispatch metrics/Cold instance-dispatch bench
128c2780b8 is described below
commit 128c2780b8564d0c14f1dc8e5c790ef156a33443
Author: Paul King <pa...@asert.com.au>
AuthorDate: Sun Jul 5 10:01:34 2026 +1000
GROOVY-12127: Improve benchmark framework: Runner calibration/Deterministic dispatch metrics/Cold instance-dispatch bench
---
.github/workflows/groovy-build-performance.yml | 39 +++-
.github/workflows/groovy-perf-daily.yml | 23 +++
.../groovy/org.apache.groovy-performance.gradle | 49 +++++
.../groovy/gradle/DispatchMetricsSummary.groovy | 103 ++++++++++
subprojects/performance/README.adoc | 56 ++++++
subprojects/performance/dashboard/jmh-summary.html | 4 +
subprojects/performance/dashboard/jmh-summary.py | 131 ++++++++++---
.../org/apache/groovy/bench/CalibrationBench.java | 30 +++
.../apache/groovy/bench/DynamicDispatchCold.groovy | 100 ++++++++++
.../groovy/bench/DynamicDispatchColdBench.java | 145 ++++++++++++++
.../calibration/AbstractCalibrationBench.java | 156 ++++++++++++++++
.../org/apache/groovy/perf/CalibrationBench.java | 30 +++
.../apache/groovy/perf/HostCalibrationBench.java | 30 +++
.../groovy/perf/grails/CalibrationBench.java | 30 +++
.../groovy/perf/grails/HostCalibrationBench.java | 30 +++
.../perf/metrics/DispatchMetricsWorkload.groovy | 208 +++++++++++++++++++++
16 files changed, 1139 insertions(+), 25 deletions(-)
diff --git a/.github/workflows/groovy-build-performance.yml b/.github/workflows/groovy-build-performance.yml
index 33302d63d1..92362c15c1 100644
--- a/.github/workflows/groovy-build-performance.yml
+++ b/.github/workflows/groovy-build-performance.yml
@@ -28,6 +28,8 @@ concurrency:
permissions:
contents: read
+ pull-requests: write # dispatch-metrics regression comments
+ issues: write
jobs:
test:
@@ -54,8 +56,43 @@ jobs:
- name: Compiler performance comparison
run: ./gradlew perf:performanceTests
timeout-minutes: 60
+
+ # Deterministic dispatch metrics (class-loading and bytecode-size
+ # counters). Unlike the timing benchmarks these are exact for a given
+ # JDK + Groovy build, so comparing against history published from
+ # *different* runner hardware is sound and the threshold can be tight.
+ - name: Dispatch metrics
+ run: ./gradlew perf:dispatchMetrics
+ timeout-minutes: 20
+
+ - name: Fetch dispatch metrics baseline
+ run: |
+ URL="https://apache.github.io/groovy/dev/bench/perf/dispatch/data.js"
+ if curl -fsSL "$URL" -o dispatch-data.js; then
+ sed -e 's/^window\.BENCHMARK_DATA = //' -e 's/;[[:space:]]*$//' dispatch-data.js > dispatch-prev.json
+ else
+ echo '{}' > dispatch-prev.json
+ fi
+
+ - name: Compare dispatch metrics against history
+ uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1
+ with:
+ name: 'Dispatch Metrics'
+ tool: 'customSmallerIsBetter'
+ output-file-path: subprojects/performance/build/dispatch-metrics.json
+ external-data-json-path: ./dispatch-prev.json
+ save-data-file: false
+ summary-always: true
+ # fork PRs get a read-only token; commenting there would 403 and fail the step
+ comment-on-alert: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+ alert-threshold: '110%'
+ fail-on-alert: false
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+
- name: Upload perf report
uses: actions/upload-artifact@v7
with:
name: perf-reports
- path: subprojects/performance/build/*.csv
+ path: |
+ subprojects/performance/build/*.csv
+ subprojects/performance/build/dispatch-metrics.json
diff --git a/.github/workflows/groovy-perf-daily.yml b/.github/workflows/groovy-perf-daily.yml
index eb85724aa0..b9f873ae6e 100644
--- a/.github/workflows/groovy-perf-daily.yml
+++ b/.github/workflows/groovy-perf-daily.yml
@@ -73,6 +73,29 @@ jobs:
fail-on-alert: false
github-token: ${{ secrets.GITHUB_TOKEN }}
+ # Deterministic dispatch metrics: class-loading and bytecode-size
+ # counters from a fixed cold-dispatch workload. Values are exact for a
+ # given JDK + Groovy build (no timing), so they are comparable across
+ # runner hardware and get a tight alert threshold.
+ # Dashboard data: dev/bench/perf/dispatch/data.js on gh-pages.
+ - name: Dispatch metrics
+ run: ./gradlew :perf:dispatchMetrics
+ timeout-minutes: 20
+
+ - name: Publish dispatch metrics
+ uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1
+ with:
+ name: 'Dispatch Metrics'
+ tool: 'customSmallerIsBetter'
+ output-file-path: subprojects/performance/build/dispatch-metrics.json
+ benchmark-data-dir-path: dev/bench/perf/dispatch
+ gh-pages-branch: gh-pages
+ auto-push: true
+ comment-always: true
+ alert-threshold: '110%'
+ fail-on-alert: false
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+
# The action writes a default index.html on first run and leaves it in
# place after that. We overlay our custom (4-series, normalised) page
# every run, but only commit if it has actually changed.
diff --git a/build-logic/src/main/groovy/org.apache.groovy-performance.gradle b/build-logic/src/main/groovy/org.apache.groovy-performance.gradle
index b57422bed4..98c8ceaa6a 100644
--- a/build-logic/src/main/groovy/org.apache.groovy-performance.gradle
+++ b/build-logic/src/main/groovy/org.apache.groovy-performance.gradle
@@ -17,6 +17,7 @@
* under the License.
*/
+import org.apache.groovy.gradle.DispatchMetricsSummary
import org.apache.groovy.gradle.PerformanceTestsExtension
import org.apache.groovy.gradle.PerformanceTestSummary
@@ -42,6 +43,7 @@ repositories {
}
dependencies {
+ implementation project(':')
jmh project(':')
testImplementation project(':')
testImplementation "org.junit.jupiter:junit-jupiter-api:${versions.junit6}"
@@ -82,6 +84,11 @@ tasks.named('compileJmhGroovy') {
inputs.property('indy', indy)
}
+tasks.named('compileGroovy') {
+ groovyOptions.optimizationOptions.indy = indy
+ inputs.property('indy', indy)
+}
+
tasks.register('displayJmhResults') {
dependsOn('jmh')
doLast {
@@ -91,6 +98,48 @@ tasks.register('displayJmhResults') {
tasks.register('performanceTests', PerformanceTestSummary)
+// Deterministic dispatch metrics: run a fixed cold-dispatch workload in a
+// fresh JVM with class+load logging, then reduce the log plus the workload's
+// own bytecode-size figures to exact, runner-independent counters. See
+// DispatchMetricsSummary and DispatchMetricsWorkload for details.
+def dispatchMetricsLog = layout.buildDirectory.file('dispatch-metrics/classload.log')
+def dispatchWorkloadJson = layout.buildDirectory.file('dispatch-metrics/workload-metrics.json')
+// dependency-free subset of the compiler-perf corpus (the other pleac files
+// need third-party libraries on the compile classpath)
+def dispatchCorpus = files(
+ 'src/files/script.groovy',
+ 'src/files/pleac02.groovy',
+ 'src/files/pleac03.groovy',
+ 'src/files/pleac04.groovy',
+ 'src/jmh/groovy/org/apache/groovy/bench/Ackermann.groovy',
+ 'src/jmh/groovy/org/apache/groovy/bench/Ary.groovy',
+ 'src/jmh/groovy/org/apache/groovy/bench/Fibo.groovy')
+
+def dispatchMetricsRun = tasks.register('dispatchMetricsRun', JavaExec) {
+ description = 'Runs the deterministic dispatch workload with class-load logging'
+ group = 'Performance'
+ classpath = sourceSets.main.runtimeClasspath
+ mainClass = 'org.apache.groovy.perf.metrics.DispatchMetricsWorkload'
+ inputs.files(dispatchCorpus)
+ outputs.file(dispatchMetricsLog)
+ outputs.file(dispatchWorkloadJson)
+ def logFile = dispatchMetricsLog.get().asFile
+ jvmArgs "-Xlog:class+load=info:file=${logFile.absolutePath}:none"
+ args([dispatchWorkloadJson.get().asFile.absolutePath] + dispatchCorpus.files.sort()*.absolutePath)
+ doFirst {
+ logFile.parentFile.mkdirs()
+ logFile.delete()
+ }
+}
+
+tasks.register('dispatchMetrics', DispatchMetricsSummary) {
+ description = 'Summarises deterministic dispatch metrics (class-load and bytecode-size counters)'
+ group = 'Performance'
+ dependsOn dispatchMetricsRun
+ classLoadLog = dispatchMetricsLog
+ workloadJson = dispatchWorkloadJson
+}
+
sonarqube {
skipProject = true
}
diff --git a/build-logic/src/main/groovy/org/apache/groovy/gradle/DispatchMetricsSummary.groovy b/build-logic/src/main/groovy/org/apache/groovy/gradle/DispatchMetricsSummary.groovy
new file mode 100644
index 0000000000..d002b70943
--- /dev/null
+++ b/build-logic/src/main/groovy/org/apache/groovy/gradle/DispatchMetricsSummary.groovy
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.groovy.gradle
+
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+import org.gradle.api.DefaultTask
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.tasks.InputFile
+import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.PathSensitive
+import org.gradle.api.tasks.PathSensitivity
+import org.gradle.api.tasks.TaskAction
+
+/**
+ * Turns the artefacts of the dispatch-metrics workload run into a single
+ * {@code customSmallerIsBetter} JSON report for the benchmark dashboards.
+ * <p>
+ * Inputs are the {@code -Xlog:class+load} log written by the forked workload
+ * JVM and the workload's own JSON fragment (bytecode-size metrics). The log
+ * is reduced to deterministic class-loading counters:
+ * <ul>
+ * <li>{@code classes.loaded.total} — every class the JVM loaded to run the
+ * workload (startup + dispatch breadth),</li>
+ * <li>{@code classes.lambdaForms} — spun {@code java.lang.invoke.LambdaForm$*}
+ * classes, the one-time MethodHandle cost of the indy machinery,</li>
+ * <li>{@code classes.hidden} — hidden classes (names carry a {@code /0x}
+ * suffix): LambdaForms plus lambda proxies and other spun adapters,</li>
+ * <li>{@code classes.groovyRuntime} — classes from Groovy's own runtime
+ * packages, the runtime's static footprint.</li>
+ * </ul>
+ * All values are exact for a given JDK + Groovy build (no timing involved),
+ * so unlike the JMH numbers they are directly comparable across different
+ * CI runner hardware; the dashboards can use tight alert thresholds.
+ */
+class DispatchMetricsSummary extends DefaultTask {
+
+ @InputFile
+ @PathSensitive(PathSensitivity.RELATIVE)
+ final RegularFileProperty classLoadLog = project.objects.fileProperty()
+
+ @InputFile
+ @PathSensitive(PathSensitivity.RELATIVE)
+ final RegularFileProperty workloadJson = project.objects.fileProperty()
+
+ @OutputFile
+ final RegularFileProperty jsonReport = project.objects.fileProperty()
+ .convention(project.layout.buildDirectory.file('dispatch-metrics.json'))
+
+ @TaskAction
+ void summarize() {
+ long total = 0
+ long lambdaForms = 0
+ long hidden = 0
+ long groovyRuntime = 0
+
+ classLoadLog.get().asFile.eachLine { line ->
+ // decorators are disabled (:none), so lines look like
+ // java.lang.invoke.LambdaForm$MH/0x0000... source: __JVM_LookupDefineClass__
+ int sourceIdx = line.indexOf(' source: ')
+ if (sourceIdx < 0) return
+ String name = line.substring(0, sourceIdx).trim()
+ if (!name || name.contains(' ')) return
+ total++
+ if (name.startsWith('java.lang.invoke.LambdaForm$')) lambdaForms++
+ if (name.contains('/0x')) hidden++
+ if (name.startsWith('org.codehaus.groovy.') || name.startsWith('org.apache.groovy.')
+ || name.startsWith('groovy.') || name.startsWith('groovyjarjar')) {
+ groovyRuntime++
+ }
+ }
+
+ def entries = [
+ [name: 'classes.loaded.total', unit: 'classes', value: total],
+ [name: 'classes.lambdaForms', unit: 'classes', value: lambdaForms],
+ [name: 'classes.hidden', unit: 'classes', value: hidden],
+ [name: 'classes.groovyRuntime', unit: 'classes', value: groovyRuntime],
+ ]
+ entries += (List) new JsonSlurper().parse(workloadJson.get().asFile)
+
+ entries.each { println "${it.name}: ${it.value} ${it.unit}" }
+
+ def out = jsonReport.get().asFile
+ out.parentFile.mkdirs()
+ out.text = JsonOutput.prettyPrint(JsonOutput.toJson(entries))
+ }
+}
diff --git a/subprojects/performance/README.adoc b/subprojects/performance/README.adoc
index 04c9a28f1f..0465b7caa0 100644
--- a/subprojects/performance/README.adoc
+++ b/subprojects/performance/README.adoc
@@ -77,3 +77,59 @@ untouched. Run them locally with:
./gradlew -PbenchInclude='\.adhoc\.' -PjmhProfilers=gc :perf:jmh
+
+== Calibration rulers
+
+Per-PR and per-commit benchmark runs are compared against a trailing 90-day
+baseline published on gh-pages — numbers produced on *different* shared-runner
+hardware, which is the dominant noise source in those comparisons. The
+`*CalibrationBench` classes are pure-Java, Groovy-independent "rulers"
+(integer throughput, pointer-chasing, allocation churn): their
+current-vs-history ratio measures the hardware delta of the current runner
+rather than anything about Groovy.
+
+Each CI suite job runs on its own runner, so one thin subclass of
+`org.apache.groovy.calibration.AbstractCalibrationBench` is named into each
+suite's `benchInclude` pattern. `dashboard/jmh-summary.py` computes a per-suite
+calibration factor from the rulers (plus the pre-existing `_java` baseline
+rows), reports a hardware-normalised "calibrated" speedup column alongside the
+raw one, and flags runs whose runner deviates more than 15% from the baseline
+hardware. The rulers are excluded from the speedup geomeans themselves, both
+in the PR summary and in the daily dashboard (`dashboard/jmh-summary.html`).
+
+== Deterministic dispatch metrics
+
+Timing benchmarks cannot give exact answers on shared runners; class-loading
+counts can. The `dispatchMetrics` task runs a fixed, deterministic
+cold-dispatch workload (`org.apache.groovy.perf.metrics.DispatchMetricsWorkload`)
+in a fresh JVM with `-Xlog:class+load` and reduces the log to counters that
+are *bit-identical* for a given JDK + Groovy build:
+
+* `classes.loaded.total` — full class-loading breadth of the workload,
+* `classes.lambdaForms` — spun `java.lang.invoke.LambdaForm$*` classes, the
+ one-time MethodHandle cost of the invokedynamic implementation (the main
+ cold-path cost driver),
+* `classes.hidden` — hidden classes (LambdaForms, lambda proxies, spun adapters),
+* `classes.groovyRuntime` — Groovy's own runtime class footprint,
+* `bytecode.corpus.bytes` / `bytecode.corpus.classes` — generated-code size for
+ compiling a fixed dependency-free corpus (inlining budgets are
+ bytecode-size based).
+
+Because the values are deterministic, they are comparable across runner
+hardware: the `perf-daily` workflow publishes them to
+`dev/bench/perf/dispatch/` on gh-pages and the per-PR `Performance check`
+workflow compares against that history with a tight (110%) alert threshold.
+Run locally with:
+
+ ./gradlew :perf:dispatchMetrics
+
+== Cold-path benchmarks
+
+`StaticMethodCallIndyColdBench` and `DynamicDispatchColdBench` measure fresh
+call sites in fresh JVMs (`SingleShotTime`, zero warmup, fork-per-sample),
+covering the first ~1000 invocations that the steady-state benchmarks warm
+away — where much real-world Groovy cost lives (scripts, CLIs, test
+bootstrap). The `n` parameter probes below and past the indy promotion
+threshold (`groovy.indy.optimize.threshold`, default 1000). Static and
+instance dispatch take different cold paths (static sites promote on first
+hit, GROOVY-11935), hence the two benches.
diff --git a/subprojects/performance/dashboard/jmh-summary.html b/subprojects/performance/dashboard/jmh-summary.html
index 1d72c4642e..ffcf228978 100644
--- a/subprojects/performance/dashboard/jmh-summary.html
+++ b/subprojects/performance/dashboard/jmh-summary.html
@@ -142,6 +142,10 @@
const cur = runs[i];
const logs = [];
for (const name in cur.benches) {
+ // Calibration rulers measure the CI runner, not Groovy — keep them
+ // out of the speedup geomean (they exist for jmh-summary.py's
+ // per-run hardware calibration).
+ if (name.indexOf('Calibration') !== -1) continue;
const b = cur.benches[name];
const hist = [];
for (let j = 0; j < i; j++) {
diff --git a/subprojects/performance/dashboard/jmh-summary.py b/subprojects/performance/dashboard/jmh-summary.py
index c4cce54b43..5e4fc4c902 100644
--- a/subprojects/performance/dashboard/jmh-summary.py
+++ b/subprojects/performance/dashboard/jmh-summary.py
@@ -20,6 +20,15 @@
# 90-day mean from gh-pages, then geomean within bench/core/grails. Mirrors the same
# normalisation as subprojects/performance/dashboard/jmh-summary.html so the numbers
# are directly comparable with the daily dashboard.
+#
+# Runner calibration: the historical baseline was produced on different (shared)
+# runner hardware, which is the dominant noise source in these comparisons. Pure-Java
+# "ruler" benchmarks (the *Calibration* benches, plus the pre-existing `_java`
+# baseline rows) are Groovy-independent, so their current-vs-history ratio measures
+# the hardware delta of this run's runner. Each CI-split part runs on its own runner,
+# so a calibration factor (geomean of ruler ratios) is computed per part and used to
+# (a) produce a hardware-normalised "calibrated" speedup column and (b) flag runs
+# whose hardware deviates so far from the baseline that raw ratios are meaningless.
import argparse
import json
@@ -35,6 +44,11 @@ from pathlib import Path
DAY_MS = 86_400_000
WINDOW_MS = 90 * DAY_MS
+# runner speed within this factor of the historical baseline counts as comparable
+CALIBRATION_TOLERANCE = 1.15
+# a calibration factor needs at least this many ruler measurements to be trusted
+MIN_RULERS = 3
+
# (group label, CI-split parts whose data.js makes up that group)
GROUPS = [
('bench', ['bench']),
@@ -54,6 +68,32 @@ def normalize(value, baseline, unit):
return value / baseline if is_higher_better(unit) else baseline / value
+def is_ruler(name):
+ """Groovy-independent benchmarks usable as host-speed rulers.
+
+ Calibration benches exist for exactly this purpose; the `_java`/`java`
+ baseline rows of the comparative benches are equally Groovy-independent
+ and give the `bench` suite ruler history predating the Calibration benches.
+ """
+ if 'Calibration' in name:
+ return True
+ method = name.split(' (')[0].rsplit('.', 1)[-1]
+ return method == 'java' or method.endswith('_java')
+
+
+def is_calibration(name):
+ return 'Calibration' in name
+
+
+def result_key(name, params):
+ """Reconstruct the name github-action-benchmark's jmh extractor publishes:
+ parameterised benchmarks become `fqn ( {"n":"500"} )`."""
+ if not params:
+ return name
+ encoded = json.dumps({k: str(v) for k, v in params.items()}, separators=(',', ':'))
+ return f'{name} ( {encoded} )'
+
+
def load_results(results_dir):
out = {}
for p in sorted(Path(results_dir).rglob('results-*.json')):
@@ -72,7 +112,7 @@ def load_results(results_dir):
value = metric.get('score')
unit = metric.get('scoreUnit', '') or ''
if name and isinstance(value, (int, float)) and value > 0:
- bench_map[name] = (float(value), unit)
+ bench_map[result_key(name, entry.get('params'))] = (float(value), unit)
if bench_map:
out[m.group(1)] = bench_map
return out
@@ -113,45 +153,88 @@ def geomean(values):
return math.exp(sum(math.log(v) for v in values) / len(values)) if values else None
+def part_ratios(current, means):
+ """Per-benchmark current-vs-baseline ratios for one CI-split part."""
+ ratios = []
+ for name, (value, unit) in current.items():
+ base = means.get(name)
+ if not base or base <= 0:
+ continue
+ r = normalize(value, base, unit)
+ if r > 0 and math.isfinite(r):
+ ratios.append((name, r))
+ return ratios
+
+
def compute_group_scores(results_by_part, mode, now_ms):
- rows = []
+ """Returns (rows, calibrations):
+ rows - (label, raw_geomean, calibrated_geomean, n) per group
+ calibrations - (part, factor_or_None, ruler_count) per part that had results
+ """
+ rows, calibrations = [], []
for label, parts in GROUPS:
- ratios = []
+ raw, calibrated = [], []
for part in parts:
current = results_by_part.get(part) or {}
if not current:
continue
means = baseline_means(fetch_baseline(part, mode), now_ms, WINDOW_MS)
- for name, (value, unit) in current.items():
- base = means.get(name)
- if not base or base <= 0:
- continue
- r = normalize(value, base, unit)
- if r > 0 and math.isfinite(r):
- ratios.append(r)
- rows.append((label, geomean(ratios), len(ratios)))
- return rows
-
-
-def render_markdown(rows, mode, commit_sha, marker):
+ ratios = part_ratios(current, means)
+ ruler_vals = [r for name, r in ratios if is_ruler(name)]
+ calib = geomean(ruler_vals) if len(ruler_vals) >= MIN_RULERS else None
+ calibrations.append((part, calib, len(ruler_vals)))
+ # Calibration rulers measure the runner, not Groovy: keep them out
+ # of the speedup scores. The legacy `_java` rows stay in the raw
+ # score (dashboard continuity) but not in the calibrated one, where
+ # they would converge to 1.0 by construction and dilute the signal.
+ raw += [r for name, r in ratios if not is_calibration(name)]
+ if calib:
+ calibrated += [r / calib for name, r in ratios if not is_ruler(name)]
+ rows.append((label, geomean(raw), geomean(calibrated), len(raw)))
+ return rows, calibrations
+
+
+def render_markdown(rows, calibrations, mode, commit_sha, marker):
short = (commit_sha or '')[:7] or 'unknown'
lines = [
f'### JMH summary — {mode} (commit `{short}`)',
'',
'Speedup vs trailing 90-day baseline on gh-pages. Higher = faster.',
'`1.00` = in line with history. Per-benchmark ratio, geomean within group.',
- 'Time-per-op units inverted so direction is consistent.',
+ 'Time-per-op units inverted so direction is consistent. The *calibrated*',
+ 'column divides out this runner\'s speed vs the baseline hardware, as',
+ 'measured by Groovy-independent pure-Java ruler benchmarks.',
'',
- '| Group | Speedup | n |',
- '|--------|---------|---|',
+ '| Group | Speedup | Calibrated | n |',
+ '|--------|---------|------------|---|',
]
any_data = False
- for label, score, n in rows:
- if score is None:
- lines.append(f'| {label} | _no overlap with baseline_ | {n} |')
+ for label, raw, calibrated, n in rows:
+ if raw is None:
+ lines.append(f'| {label} | _no overlap with baseline_ | — | {n} |')
else:
any_data = True
- lines.append(f'| {label} | {score:.3f} × | {n} |')
+ cal = f'{calibrated:.3f} ×' if calibrated is not None else '—'
+ lines.append(f'| {label} | {raw:.3f} × | {cal} | {n} |')
+
+ with_calib = [(part, factor, count) for part, factor, count in calibrations if factor is not None]
+ off_parts = [part for part, factor, _ in with_calib
+ if factor > CALIBRATION_TOLERANCE or factor < 1 / CALIBRATION_TOLERANCE]
+ if off_parts:
+ lines += [
+ '',
+ f'> ⚠️ Runner speed differs ≥{round((CALIBRATION_TOLERANCE - 1) * 100)}% from the '
+ f'historical baseline hardware for: {", ".join(off_parts)}. '
+ 'Raw speedups are not meaningful for those parts — use the calibrated column.',
+ ]
+ if with_calib:
+ calib_text = ' · '.join(f'{part} {factor:.2f}× ({count} rulers)'
+ for part, factor, count in with_calib)
+ lines += ['', f'<sub>Runner calibration (this run vs baseline hardware): {calib_text}</sub>']
+ else:
+ lines += ['', '<sub>Runner calibration: no ruler history yet (Calibration '
+ 'benches need to appear in the gh-pages baseline first).</sub>']
+
lines += [
'',
f'<sub>Baseline: <code>dev/bench/jmh/<part>/{mode}/data.js</code> on '
@@ -213,9 +296,9 @@ def main():
print(f'No results-*.json found under {args.results_dir}', file=sys.stderr)
return 1
- rows = compute_group_scores(results_by_part, args.mode, int(time.time() * 1000))
+ rows, calibrations = compute_group_scores(results_by_part, args.mode, int(time.time() * 1000))
marker = f'<!-- jmh-summary:{args.mode} -->'
- body, any_data = render_markdown(rows, args.mode, args.commit, marker)
+ body, any_data = render_markdown(rows, calibrations, args.mode, args.commit, marker)
print(body)
pr = (args.pr_number or '').strip()
diff --git a/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/CalibrationBench.java b/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/CalibrationBench.java
new file mode 100644
index 0000000000..70fcfdfbe2
--- /dev/null
+++ b/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/CalibrationBench.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.groovy.bench;
+
+import org.apache.groovy.calibration.AbstractCalibrationBench;
+
+/**
+ * Host-speed calibration ruler for the {@code bench} CI suite; the class
+ * name/package exist solely to match that suite's {@code benchInclude}
+ * pattern ({@code \.bench\.}). See {@link AbstractCalibrationBench} for
+ * what is measured and why.
+ */
+public class CalibrationBench extends AbstractCalibrationBench {
+}
diff --git a/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/DynamicDispatchCold.groovy b/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/DynamicDispatchCold.groovy
new file mode 100644
index 0000000000..3849b63ca5
--- /dev/null
+++ b/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/DynamicDispatchCold.groovy
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.groovy.bench
+
+import groovy.transform.CompileStatic
+
+/**
+ * Dynamic-dispatch fixture for {@link DynamicDispatchColdBench}. All methods
+ * except the {@code CS} variant are deliberately dynamic so each call site
+ * goes through the full indy (or classic call-site) machinery from a cold
+ * start: bootstrap, method selection, guard construction, and — depending on
+ * {@code n} — promotion past the hit-count threshold.
+ */
+class DynamicDispatchCold {
+
+ static class Alpha {
+ int work(int i) { i + 1 }
+ int getVal() { 1 }
+ }
+
+ static class Beta extends Alpha {
+ int work(int i) { i + 2 }
+ int getVal() { 2 }
+ }
+
+ static class Gamma extends Alpha {
+ int work(int i) { i + 3 }
+ int getVal() { 3 }
+ }
+
+ /** Monomorphic dynamic instance calls: one receiver type at the call site. */
+ int monoSum(int n) {
+ def r = new Alpha()
+ int s = 0
+ for (int i = 0; i < n; i++) {
+ s += r.work(i)
+ }
+ s
+ }
+
+ /** Polymorphic dynamic instance calls: three receiver types rotate at one call site. */
+ int polySum(int n) {
+ def a = new Alpha()
+ def b = new Beta()
+ def c = new Gamma()
+ int s = 0
+ for (int i = 0; i < n; i++) {
+ def r = (i % 3 == 0) ? a : ((i % 3 == 1) ? b : c)
+ s += r.work(i)
+ }
+ s
+ }
+
+ /** Dynamic property reads through the MOP. */
+ int propertySum(int n) {
+ def r = new Beta()
+ int s = 0
+ for (int i = 0; i < n; i++) {
+ s += r.val
+ }
+ s
+ }
+
+ /** Dynamic closure invocation. */
+ int closureSum(int n) {
+ def f = { int i -> i + 1 }
+ int s = 0
+ for (int i = 0; i < n; i++) {
+ s += f(i)
+ }
+ s
+ }
+
+ /** Statically-compiled lower bound for {@link #monoSum}. */
+ @CompileStatic
+ int monoSumCS(int n) {
+ Alpha r = new Alpha()
+ int s = 0
+ for (int i = 0; i < n; i++) {
+ s += r.work(i)
+ }
+ s
+ }
+}
diff --git a/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/DynamicDispatchColdBench.java b/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/DynamicDispatchColdBench.java
new file mode 100644
index 0000000000..ba8c03c94b
--- /dev/null
+++ b/subprojects/performance/src/jmh/groovy/org/apache/groovy/bench/DynamicDispatchColdBench.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.groovy.bench;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Cold-start counterpart to the steady-state dispatch benchmarks — measures
+ * fresh <em>instance</em>-dispatch call sites (dynamic method calls, property
+ * access, closure invocation) before they reach steady state, complementing
+ * {@link StaticMethodCallIndyColdBench} which covers only static calls.
+ * <p>
+ * Instance dispatch takes a different cold path than static dispatch: static
+ * indy call sites get their target installed on the first hit (GROOVY-11935),
+ * while instance sites stay on the bootstrap adapter until the hit-count
+ * threshold ({@code groovy.indy.optimize.threshold}, default 1000) promotes
+ * them. The {@code n} parameter probes both regimes:
+ * <ul>
+ * <li>{@code n=500} — entirely below the threshold; every invocation runs
+ * the cached-handle lookup path,</li>
+ * <li>{@code n=20000} — well past it; the call site promotes early and the
+ * result approaches steady state.</li>
+ * </ul>
+ * {@link Mode#SingleShotTime} with zero warmup and a fresh JVM per fork; the
+ * mean over forks estimates true cold cost. The Java and
+ * {@code @CompileStatic} rows are direct-dispatch lower bounds. The fork
+ * count is a compromise between sampling error and CI wall-clock (the suite
+ * job has a 60-minute budget shared with the other cold bench).
+ * <p>
+ * Useful for guarding {@code IndyInterface}/{@code Selector}/
+ * {@code CacheableCallSite} changes that alter cold instance dispatch —
+ * polymorphic-inline-cache reworks, promotion-threshold changes, boot-handle
+ * simplifications — which the throughput benches cannot see.
+ */
+@Warmup(iterations = 0)
+@Measurement(iterations = 1, batchSize = 1)
+@Fork(25)
+@BenchmarkMode(Mode.SingleShotTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Thread)
+public class DynamicDispatchColdBench {
+
+ /**
+ * Iteration count probing below (500) and past (20000) the indy
+ * hit-count promotion threshold.
+ */
+ @Param({"500", "20000"})
+ public int n;
+
+ /**
+ * Monomorphic dynamic instance calls from a cold call site.
+ * @return the computed sum
+ */
+ @Benchmark
+ public int dynamicMono_groovy() {
+ return new DynamicDispatchCold().monoSum(n);
+ }
+
+ /**
+ * Polymorphic (3 receiver types) dynamic instance calls from a cold call site.
+ * @return the computed sum
+ */
+ @Benchmark
+ public int dynamicPoly_groovy() {
+ return new DynamicDispatchCold().polySum(n);
+ }
+
+ /**
+ * Dynamic property reads from a cold call site.
+ * @return the computed sum
+ */
+ @Benchmark
+ public int property_groovy() {
+ return new DynamicDispatchCold().propertySum(n);
+ }
+
+ /**
+ * Dynamic closure invocation from a cold call site.
+ * @return the computed sum
+ */
+ @Benchmark
+ public int closure_groovy() {
+ return new DynamicDispatchCold().closureSum(n);
+ }
+
+ /**
+ * {@code @CompileStatic} lower bound for the monomorphic case.
+ * @return the computed sum
+ */
+ @Benchmark
+ public int dynamicMono_groovyCS() {
+ return new DynamicDispatchCold().monoSumCS(n);
+ }
+
+ /**
+ * Java lower bound (direct invokevirtual) for the monomorphic case.
+ * @return the computed sum
+ */
+ @Benchmark
+ public int dynamicMono_java() {
+ return javaMonoSum(n);
+ }
+
+ private static int javaMonoSum(int n) {
+ JavaAlpha r = new JavaAlpha();
+ int s = 0;
+ for (int i = 0; i < n; i++) {
+ s += r.work(i);
+ }
+ return s;
+ }
+
+ static final class JavaAlpha {
+ int work(int i) {
+ return i + 1;
+ }
+ }
+}
diff --git a/subprojects/performance/src/jmh/groovy/org/apache/groovy/calibration/AbstractCalibrationBench.java b/subprojects/performance/src/jmh/groovy/org/apache/groovy/calibration/AbstractCalibrationBench.java
new file mode 100644
index 0000000000..23c38772a7
--- /dev/null
+++ b/subprojects/performance/src/jmh/groovy/org/apache/groovy/calibration/AbstractCalibrationBench.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.groovy.calibration;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Host-speed calibration "rulers": pure-Java benchmarks that are entirely
+ * independent of Groovy, so any change in their results reflects the CI
+ * runner (hardware, JDK build, neighbours) rather than the code under test.
+ * <p>
+ * The per-commit summary ({@code dashboard/jmh-summary.py}) compares each
+ * run against the trailing 90-day history published on gh-pages. Those
+ * historical numbers come from <em>different runner hardware</em>, which is
+ * the dominant noise source in per-PR comparisons. The rulers measure that
+ * hardware delta directly: the geomean of their current-vs-history ratios is
+ * the run's <em>calibration factor</em>, used to (a) normalise the Groovy
+ * benchmark ratios and (b) flag low-confidence runs whose hardware deviates
+ * too far from the historical baseline.
+ * <p>
+ * Three rulers cover the dimensions on which shared runners differ most:
+ * <ul>
+ * <li>{@link #cpuIntegerOps()} — scalar integer throughput (core speed),</li>
+ * <li>{@link #memoryPointerChase()} — dependent loads over a 1 MiB
+ * permutation (cache/memory subsystem),</li>
+ * <li>{@link #allocationChurn()} — small-object allocation pressure
+ * (allocation/GC speed).</li>
+ * </ul>
+ * All work is deterministic (fixed seeds, fixed iteration counts) and, being
+ * plain Java, produces identical bytecode in the indy and classic suites.
+ * <p>
+ * Each CI suite job runs on its own runner, so each suite needs its own
+ * ruler. The concrete subclasses exist only to place one copy inside every
+ * suite's {@code benchInclude} pattern (see {@code .github/workflows/groovy-jmh*.yml}):
+ * <ul>
+ * <li>{@code org.apache.groovy.bench.CalibrationBench} — suite {@code bench} ({@code \.bench\.})</li>
+ * <li>{@code org.apache.groovy.perf.CalibrationBench} — suite {@code core-ag} ({@code \.perf\.[A-G]})</li>
+ * <li>{@code org.apache.groovy.perf.HostCalibrationBench} — suite {@code core-hz} ({@code \.perf\.[H-Z]})</li>
+ * <li>{@code org.apache.groovy.perf.grails.CalibrationBench} — suite {@code grails-ad} ({@code \.perf\.grails\.[A-D]})</li>
+ * <li>{@code org.apache.groovy.perf.grails.HostCalibrationBench} — suite {@code grails-ez} ({@code \.perf\.grails\.[E-Z]})</li>
+ * </ul>
+ * Keep "Calibration" in any subclass name: the summary tooling identifies
+ * rulers by that marker and excludes them from the Groovy speedup geomeans.
+ * The reduced warmup/measurement keeps the cost near one minute per suite;
+ * rulers need collective, not individual, precision.
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Warmup(iterations = 2, time = 1)
+@Measurement(iterations = 3, time = 1)
+@Fork(2)
+@State(Scope.Thread)
+public abstract class AbstractCalibrationBench {
+
+ private static final int CHAIN_SIZE = 1 << 18; // 1 MiB of ints: larger than typical L2
+ private static final int HOLD_SLOTS = 32;
+
+ private int[] chain;
+ private Object[] holder;
+
+ /**
+ * Builds a single-cycle permutation (Sattolo's algorithm with a fixed
+ * LCG) so the pointer chase visits all elements without short cycles.
+ */
+ @Setup(Level.Trial)
+ public void setup() {
+ chain = new int[CHAIN_SIZE];
+ for (int i = 0; i < CHAIN_SIZE; i++) {
+ chain[i] = i;
+ }
+ long seed = 0x9E3779B97F4A7C15L;
+ for (int i = CHAIN_SIZE - 1; i > 0; i--) {
+ seed = seed * 6364136223846793005L + 1442695040888963407L;
+ int j = (int) Math.floorMod(seed >>> 33, (long) i);
+ int tmp = chain[i];
+ chain[i] = chain[j];
+ chain[j] = tmp;
+ }
+ holder = new Object[HOLD_SLOTS];
+ }
+
+ /**
+ * Scalar integer throughput ruler (xorshift64).
+ * @return the mixed state, to defeat dead-code elimination
+ */
+ @Benchmark
+ public long cpuIntegerOps() {
+ long x = 0xDEADBEEFCAFEBABEL;
+ for (int i = 0; i < 200_000; i++) {
+ x ^= x << 13;
+ x ^= x >>> 7;
+ x ^= x << 17;
+ }
+ return x;
+ }
+
+ /**
+ * Memory-subsystem ruler: dependent loads through a 1 MiB permutation.
+ * @return the final chase index, to defeat dead-code elimination
+ */
+ @Benchmark
+ public int memoryPointerChase() {
+ int[] c = chain;
+ int idx = 0;
+ for (int i = 0; i < 100_000; i++) {
+ idx = c[idx];
+ }
+ return idx;
+ }
+
+ /**
+ * Allocation-pressure ruler: small arrays escaping into a rotating
+ * holder so escape analysis cannot scalar-replace them.
+ * @return a value derived from the allocations, to defeat dead-code elimination
+ */
+ @Benchmark
+ public long allocationChurn() {
+ long s = 0;
+ Object[] h = holder;
+ for (int i = 0; i < 10_000; i++) {
+ int[] a = new int[16];
+ a[i & 15] = i;
+ h[i & (HOLD_SLOTS - 1)] = a;
+ s += a[i & 15];
+ }
+ return s;
+ }
+}
diff --git a/subprojects/performance/src/jmh/groovy/org/apache/groovy/perf/CalibrationBench.java b/subprojects/performance/src/jmh/groovy/org/apache/groovy/perf/CalibrationBench.java
new file mode 100644
index 0000000000..58ecc83b55
--- /dev/null
+++ b/subprojects/performance/src/jmh/groovy/org/apache/groovy/perf/CalibrationBench.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.groovy.perf;
+
+import org.apache.groovy.calibration.AbstractCalibrationBench;
+
+/**
+ * Host-speed calibration ruler for the {@code core-ag} CI suite; the class
+ * name/package exist solely to match that suite's {@code benchInclude}
+ * pattern ({@code \.perf\.[A-G]}). See {@link AbstractCalibrationBench} for
+ * what is measured and why.
+ */
+public class CalibrationBench extends AbstractCalibrationBench {
+}
diff --git a/subprojects/performance/src/jmh/groovy/org/apache/groovy/perf/HostCalibrationBench.java b/subprojects/performance/src/jmh/groovy/org/apache/groovy/perf/HostCalibrationBench.java
new file mode 100644
index 0000000000..4fdef492dc
--- /dev/null
+++ b/subprojects/performance/src/jmh/groovy/org/apache/groovy/perf/HostCalibrationBench.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.groovy.perf;
+
+import org.apache.groovy.calibration.AbstractCalibrationBench;
+
+/**
+ * Host-speed calibration ruler for the {@code core-hz} CI suite; the class
+ * name/package exist solely to match that suite's {@code benchInclude}
+ * pattern ({@code \.perf\.[H-Z]}). See {@link AbstractCalibrationBench} for
+ * what is measured and why.
+ */
+public class HostCalibrationBench extends AbstractCalibrationBench {
+}
diff --git a/subprojects/performance/src/jmh/groovy/org/apache/groovy/perf/grails/CalibrationBench.java b/subprojects/performance/src/jmh/groovy/org/apache/groovy/perf/grails/CalibrationBench.java
new file mode 100644
index 0000000000..e59b517176
--- /dev/null
+++ b/subprojects/performance/src/jmh/groovy/org/apache/groovy/perf/grails/CalibrationBench.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.groovy.perf.grails;
+
+import org.apache.groovy.calibration.AbstractCalibrationBench;
+
+/**
+ * Host-speed calibration ruler for the {@code grails-ad} CI suite; the class
+ * name/package exist solely to match that suite's {@code benchInclude}
+ * pattern ({@code \.perf\.grails\.[A-D]}). See {@link AbstractCalibrationBench}
+ * for what is measured and why.
+ */
+public class CalibrationBench extends AbstractCalibrationBench {
+}
diff --git a/subprojects/performance/src/jmh/groovy/org/apache/groovy/perf/grails/HostCalibrationBench.java b/subprojects/performance/src/jmh/groovy/org/apache/groovy/perf/grails/HostCalibrationBench.java
new file mode 100644
index 0000000000..02cde5b2d4
--- /dev/null
+++ b/subprojects/performance/src/jmh/groovy/org/apache/groovy/perf/grails/HostCalibrationBench.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.groovy.perf.grails;
+
+import org.apache.groovy.calibration.AbstractCalibrationBench;
+
+/**
+ * Host-speed calibration ruler for the {@code grails-ez} CI suite; the class
+ * name/package exist solely to match that suite's {@code benchInclude}
+ * pattern ({@code \.perf\.grails\.[E-Z]}). See {@link AbstractCalibrationBench}
+ * for what is measured and why.
+ */
+public class HostCalibrationBench extends AbstractCalibrationBench {
+}
diff --git a/subprojects/performance/src/main/groovy/org/apache/groovy/perf/metrics/DispatchMetricsWorkload.groovy b/subprojects/performance/src/main/groovy/org/apache/groovy/perf/metrics/DispatchMetricsWorkload.groovy
new file mode 100644
index 0000000000..193157f1d0
--- /dev/null
+++ b/subprojects/performance/src/main/groovy/org/apache/groovy/perf/metrics/DispatchMetricsWorkload.groovy
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.groovy.perf.metrics
+
+import org.codehaus.groovy.control.CompilationUnit
+import org.codehaus.groovy.control.CompilerConfiguration
+import org.codehaus.groovy.control.Phases
+
+/**
+ * Deterministic workload behind the {@code perf:dispatchMetrics} task.
+ * <p>
+ * Timing benchmarks on shared CI runners are dominated by hardware variance,
+ * so regressions hide in the noise. This workload instead produces metrics
+ * that are <em>exact</em> for a given JDK + Groovy build and therefore
+ * directly comparable across runners and against published history:
+ * <ul>
+ * <li>classes loaded while executing a fixed set of cold dynamic-dispatch
+ * scenarios (captured externally via {@code -Xlog:class+load} and
+ * counted by the Gradle side — total, {@code LambdaForm$*}, hidden,
+ * Groovy-runtime), a direct proxy for the one-time
+ * MethodHandle/LambdaForm cost of the indy implementation;</li>
+ * <li>total bytecode bytes and class count for compiling a fixed,
+ * dependency-free source corpus — a stable measure of generated-code
+ * size (inlining budgets are bytecode-size based).</li>
+ * </ul>
+ * Everything here must stay deterministic: fixed iteration counts, fixed
+ * inputs, single-threaded execution, no time- or randomness-dependent
+ * behaviour. Scenario iteration counts stay below the indy promotion
+ * threshold ({@code groovy.indy.optimize.threshold}, default 1000) so the
+ * dispatch machinery is exercised on its cold tier, where most real-world
+ * script/CLI/test-bootstrap cost lives.
+ * <p>
+ * Usage: {@code DispatchMetricsWorkload <output-json> <corpus-file>...}
+ * The output JSON fragment uses the same
+ * {@code customSmallerIsBetter} entry schema as the final report so the two
+ * can simply be concatenated.
+ */
+class DispatchMetricsWorkload {
+
+ /** Iterations per dispatch scenario; kept below the indy promotion threshold. */
+ static final int COLD_ITERATIONS = 512
+
+ static void main(String[] args) {
+ if (args.length < 2) {
+ System.err.println 'usage: DispatchMetricsWorkload <output-json> <corpus-file>...'
+ System.exit 1
+ }
+ File outFile = new File(args[0])
+ List<File> corpus = args[1..-1].collect { new File(it) }
+
+ long checksum = runDispatchScenarios()
+ checksum += evaluateScript()
+
+ long corpusBytes = 0
+ int corpusClasses = 0
+ for (File source : corpus) {
+ def unit = new CompilationUnit(new CompilerConfiguration())
+ unit.addSource(source)
+ unit.compile(Phases.CLASS_GENERATION)
+ unit.classes.each { generated ->
+ corpusBytes += generated.bytes.length
+ corpusClasses++
+ }
+ }
+
+ outFile.parentFile?.mkdirs()
+ outFile.text = """[
+ { "name": "bytecode.corpus.bytes", "unit": "bytes", "value": ${corpusBytes} },
+ { "name": "bytecode.corpus.classes", "unit": "classes", "value": ${corpusClasses} }
+]
+"""
+ // the checksum keeps the scenario work observable (and un-eliminable)
+ println "dispatch workload complete: checksum=${checksum}, corpus=${corpusClasses} classes / ${corpusBytes} bytes"
+ }
+
+ private static long runDispatchScenarios() {
+ def scenarios = new Scenarios()
+ long checksum = 0
+ checksum += scenarios.monomorphicCalls(COLD_ITERATIONS)
+ checksum += scenarios.polymorphicCalls(COLD_ITERATIONS)
+ checksum += scenarios.propertyAccess(COLD_ITERATIONS)
+ checksum += scenarios.closureCalls(COLD_ITERATIONS)
+ checksum += scenarios.operatorCalls(COLD_ITERATIONS)
+ checksum += scenarios.gstringsAndCollections(COLD_ITERATIONS >> 1)
+ checksum += scenarios.constructorCalls(COLD_ITERATIONS >> 1)
+ checksum
+ }
+
+ /** Compile-and-run cost of a small script, the groovy-CLI/test-bootstrap pattern. */
+ private static long evaluateScript() {
+ def shell = new GroovyShell()
+ def result = shell.evaluate('''
+ class Greeter {
+ String name
+ String greet() { "hello, ${name}" }
+ }
+ def acc = 0
+ (1..64).each { acc += it }
+ def g = new Greeter(name: 'groovy')
+ acc + g.greet().size()
+ ''')
+ ((Number) result).longValue()
+ }
+
+ /**
+ * Cold dynamic-dispatch scenarios. Deliberately dynamic Groovy: every
+ * call below goes through the runtime's dispatch machinery.
+ */
+ static class Scenarios {
+
+ static class Node {
+ int weight
+ int bump(int i) { weight + i }
+ }
+
+ static class HeavyNode extends Node {
+ @Override
+ int bump(int i) { weight + 2 * i }
+ }
+
+ static class LightNode extends Node {
+ @Override
+ int bump(int i) { weight - i }
+ }
+
+ long monomorphicCalls(int n) {
+ def node = new Node(weight: 3)
+ long s = 0
+ for (int i = 0; i < n; i++) {
+ s += node.bump(i)
+ }
+ s
+ }
+
+ long polymorphicCalls(int n) {
+ def nodes = [new Node(weight: 1), new HeavyNode(weight: 2), new LightNode(weight: 3)]
+ long s = 0
+ for (int i = 0; i < n; i++) {
+ s += nodes[i % 3].bump(i)
+ }
+ s
+ }
+
+ long propertyAccess(int n) {
+ def node = new HeavyNode(weight: 5)
+ long s = 0
+ for (int i = 0; i < n; i++) {
+ node.weight = i
+ s += node.weight
+ }
+ s
+ }
+
+ long closureCalls(int n) {
+ def twice = { int x -> 2 * x }
+ def offset = 7
+ def shifted = { int x -> x + offset }
+ long s = 0
+ for (int i = 0; i < n; i++) {
+ s += twice(i) + shifted(i)
+ }
+ s
+ }
+
+ long operatorCalls(int n) {
+ def total = BigInteger.ZERO
+ def list = []
+ for (int i = 0; i < n; i++) {
+ total = total + BigInteger.valueOf(i)
+ list << i
+ }
+ total.longValue() + list.size()
+ }
+
+ long gstringsAndCollections(int n) {
+ long s = 0
+ def words = (0..<n).collect { "item-${it}" }
+ words.each { s += it.size() }
+ s += words.findAll { it.endsWith('7') }.size()
+ s
+ }
+
+ long constructorCalls(int n) {
+ long s = 0
+ for (int i = 0; i < n; i++) {
+ def node = (i % 2 == 0) ? new Node(weight: i) : new HeavyNode(weight: i)
+ s += node.weight
+ }
+ s
+ }
+ }
+}