You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2026/07/02 13:09:48 UTC
(skywalking) branch master updated: Fix false-positive DSL classloader leak warnings on idle heaps (#13935)
This is an automated email from the ASF dual-hosted git repository.
wu-sheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 1432e5eeb2 Fix false-positive DSL classloader leak warnings on idle heaps (#13935)
1432e5eeb2 is described below
commit 1432e5eeb20ce93a16978200ace289949b94b947
Author: 吴晟 Wu Sheng <wu...@foxmail.com>
AuthorDate: Thu Jul 2 21:09:43 2026 +0800
Fix false-positive DSL classloader leak warnings on idle heaps (#13935)
---
.../oap/server/core/classloader/ClassLoaderGc.java | 181 ++++++++++++++++++++-
.../core/classloader/DSLClassLoaderManager.java | 55 +++++--
.../core/classloader/UnloadProbePayload.java | 32 ++++
.../server/core/classloader/ClassLoaderGcTest.java | 96 +++++++++++
4 files changed, 342 insertions(+), 22 deletions(-)
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/ClassLoaderGc.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/ClassLoaderGc.java
index f805fd9dd5..cd2286215d 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/ClassLoaderGc.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/ClassLoaderGc.java
@@ -18,11 +18,16 @@
package org.apache.skywalking.oap.server.core.classloader;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
import java.lang.ref.PhantomReference;
import java.lang.ref.Reference;
import java.lang.ref.ReferenceQueue;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -40,18 +45,43 @@ import lombok.extern.slf4j.Slf4j;
* holding the class) will pin the loader and its classes indefinitely. Without observability,
* that kind of leak is invisible until heap pressure surfaces hours later.
*
+ * <p><b>Leak evidence, not wall-clock guessing.</b> A classloader that has defined classes is
+ * only reclaimed by a class-unloading-capable GC cycle (G1 concurrent mark / full GC — young
+ * collections never unload classes), and an idle heap may not run one for hours. So "retired
+ * N minutes ago and still uncollected" is NOT a leak signal by itself. To tell a pinned
+ * loader apart from plain GC inactivity, the graveyard arms an <em>unload probe</em> whenever
+ * entries are pending: a parent-less throwaway classloader that defines one empty class
+ * ({@link UnloadProbePayload}) and is immediately dereferenced. The probe has the exact same
+ * collection requirement as a retired rule loader, so its collection is proof that a
+ * class-unloading cycle completed after the probe was minted. {@link #leakSuspects} flags an
+ * entry only when such a cycle ran comfortably after the entry's retirement and the entry
+ * still survived it.
+ *
* <p>This graveyard is internal to {@link DSLClassLoaderManager}. The manager retires loaders
* here via {@code dropRuntime} (full teardown) and {@code retire} (engine-decided "displaced
* prior is dead"); a daemon sweeper thread the manager owns drains collected phantoms +
- * WARNs on stale entries. No external caller touches this class — every consumer goes
- * through the manager.
+ * WARNs on evidence-backed suspects. No external caller touches this class — every consumer
+ * goes through the manager.
*/
@Slf4j
final class ClassLoaderGc {
+ private static final String PROBE_PAYLOAD_CLASS = UnloadProbePayload.class.getName();
+ private static final byte[] PROBE_PAYLOAD_BYTECODE = readProbePayloadBytecode();
+
private final ReferenceQueue<RuleClassLoader> queue = new ReferenceQueue<>();
private final Map<PhantomReference<RuleClassLoader>, Retired> pending = new ConcurrentHashMap<>();
+ private final ReferenceQueue<ClassLoader> probeQueue = new ReferenceQueue<>();
+ private final Object probeLock = new Object();
+ /** Live probe generation; at most one at a time. Guarded by {@link #probeLock}. The
+ * phantom ref must stay strongly held here or it would be GC'd before enqueuing. */
+ private PhantomReference<ClassLoader> liveProbe;
+ private long liveProbeMintedAtMs;
+ /** Latest probe mint-time for which a class-unloading GC cycle is confirmed complete.
+ * {@code 0} until the first probe collection is observed. */
+ private volatile long unloadEvidenceUpToMs;
+
@Getter
private final AtomicLong collectedTotal = new AtomicLong();
@@ -69,39 +99,170 @@ final class ClassLoaderGc {
loader.getKind(), loader.getCatalog(), loader.getRule(), loader.getContentHash(),
System.currentTimeMillis(), ref);
pending.put(ref, retired);
+ armUnloadProbe();
}
/**
* Drain collected phantoms from the queue. Returns the entries the JVM confirmed as
* unreachable since the last sweep. Entries that remain in {@link #pending()} after this
- * call are still suspected leaks. Called by the manager's internal sweeper thread.
+ * call have not been collected yet — {@link #leakSuspects} decides which of them are
+ * evidence-backed leaks. Called by the manager's internal sweeper thread.
*/
Collection<Retired> sweep() {
- final java.util.ArrayList<Retired> drained = new java.util.ArrayList<>();
+ drainUnloadProbe();
+ final List<Retired> drained = new ArrayList<>();
Reference<?> r;
while ((r = queue.poll()) != null) {
final Retired done = pending.remove(r);
if (done != null) {
collectedTotal.incrementAndGet();
- log.info("rule loader collected: {}:{}/{} hash={} ttg={}ms",
+ log.info("rule loader collected: {}:{}/{} hash={} ttg={}ms{}",
done.kind() == DSLClassLoaderManager.Kind.BUNDLED ? "bundled" : "runtime-rule",
done.catalog().getWireName(), done.rule(), done.contentHashShort(),
- System.currentTimeMillis() - done.retiredAtMs());
+ System.currentTimeMillis() - done.retiredAtMs(),
+ done.warnedAlready()
+ ? " — the earlier leak warning is cleared, the lingering reference has been released"
+ : "");
drained.add(done);
}
}
+ // Re-arm so entries retired after the previous probe was minted get their own
+ // evidence generation on a later class-unloading cycle.
+ if (!pending.isEmpty()) {
+ armUnloadProbe();
+ }
return drained;
}
/**
* @return snapshot of retired-but-not-yet-GC'd entries. Elevated steadily across many
- * sweeps == leak; the manager's sweeper logs WARN per entry older than the
- * configured threshold.
+ * sweeps == leak; the manager's sweeper logs WARN per entry that
+ * {@link #leakSuspects} confirms against unload evidence.
*/
Collection<Retired> pending() {
return Collections.unmodifiableCollection(pending.values());
}
+ /**
+ * Entries whose leak WARN should fire now: a class-unloading GC cycle is confirmed to
+ * have completed at least {@code settleMs} after the entry's retirement, and the entry
+ * survived it — so something still strongly references the loader; GC inactivity is
+ * ruled out. The settle window keeps legitimately transient holders (an apply call
+ * chain still carrying the displaced bundle on a request thread while the cycle ran)
+ * from tripping the detector. Each entry is returned at most once across the graveyard's
+ * lifetime (single-shot {@code warned} latch).
+ *
+ * <p>If the probe payload bytecode is unavailable (unreadable resource — effectively
+ * never), evidence can't be produced; the check degrades to the wall-clock heuristic
+ * rather than never warning at all.
+ */
+ Collection<Retired> leakSuspects(final long settleMs) {
+ final List<Retired> out = new ArrayList<>();
+ final long evidenceUpToMs = unloadEvidenceUpToMs;
+ final long nowMs = System.currentTimeMillis();
+ for (final Retired r : pending.values()) {
+ final boolean suspect = PROBE_PAYLOAD_BYTECODE == null
+ ? nowMs - r.retiredAtMs() > settleMs
+ : evidenceUpToMs >= r.retiredAtMs() + settleMs;
+ if (suspect && r.markWarnedIfNotAlready()) {
+ out.add(r);
+ }
+ }
+ return out;
+ }
+
+ /** Latest probe mint-time confirmed survived-past by a class-unloading GC cycle;
+ * {@code 0} while no cycle has been observed. Exposed for the manager's diagnostics
+ * and for deterministic tests. */
+ long unloadEvidenceUpToMs() {
+ return unloadEvidenceUpToMs;
+ }
+
+ /** Whether the unload probe can produce evidence. {@code false} only when the payload
+ * bytecode resource is unreadable — {@link #leakSuspects} then degrades to the
+ * wall-clock heuristic, and the manager's WARN wording must not claim GC evidence. */
+ boolean unloadProbeUsable() {
+ return PROBE_PAYLOAD_BYTECODE != null;
+ }
+
+ /**
+ * Record that a class-unloading GC cycle completed after {@code probeMintedAtMs}.
+ * Normally driven by {@link #drainUnloadProbe()}; package-private so tests can exercise
+ * {@link #leakSuspects} without depending on real GC timing.
+ */
+ void recordUnloadEvidence(final long probeMintedAtMs) {
+ synchronized (probeLock) {
+ if (probeMintedAtMs > unloadEvidenceUpToMs) {
+ unloadEvidenceUpToMs = probeMintedAtMs;
+ }
+ }
+ }
+
+ /** Mint a fresh probe if none is live. The probe loader leaves this method with the
+ * phantom reference as its only remaining reference, so the next class-unloading GC
+ * cycle is guaranteed to collect it. */
+ private void armUnloadProbe() {
+ if (PROBE_PAYLOAD_BYTECODE == null) {
+ return;
+ }
+ synchronized (probeLock) {
+ if (liveProbe != null) {
+ return;
+ }
+ final ProbeClassLoader probe = new ProbeClassLoader();
+ probe.definePayload();
+ liveProbe = new PhantomReference<>(probe, probeQueue);
+ liveProbeMintedAtMs = System.currentTimeMillis();
+ }
+ }
+
+ private void drainUnloadProbe() {
+ boolean collected = false;
+ while (probeQueue.poll() != null) {
+ collected = true;
+ }
+ if (collected) {
+ synchronized (probeLock) {
+ recordUnloadEvidence(liveProbeMintedAtMs);
+ liveProbe = null;
+ }
+ }
+ }
+
+ private static byte[] readProbePayloadBytecode() {
+ final String resource = "/" + PROBE_PAYLOAD_CLASS.replace('.', '/') + ".class";
+ try (InputStream in = ClassLoaderGc.class.getResourceAsStream(resource)) {
+ if (in == null) {
+ log.warn("unload-probe payload bytecode not found at {}; "
+ + "loader leak detection degrades to the wall-clock heuristic", resource);
+ return null;
+ }
+ final ByteArrayOutputStream out = new ByteArrayOutputStream(512);
+ final byte[] buf = new byte[1024];
+ int n;
+ while ((n = in.read(buf)) > 0) {
+ out.write(buf, 0, n);
+ }
+ return out.toByteArray();
+ } catch (final IOException e) {
+ log.warn("failed to read unload-probe payload bytecode; "
+ + "loader leak detection degrades to the wall-clock heuristic", e);
+ return null;
+ }
+ }
+
+ /** Parent-less loader whose single defined class subjects it to class-unloading
+ * collection semantics. Never asked to load anything. */
+ private static final class ProbeClassLoader extends ClassLoader {
+ ProbeClassLoader() {
+ super(null);
+ }
+
+ void definePayload() {
+ defineClass(PROBE_PAYLOAD_CLASS, PROBE_PAYLOAD_BYTECODE, 0, PROBE_PAYLOAD_BYTECODE.length);
+ }
+ }
+
/** Informational record surfaced to the sweeper. Identity is immutable; only the
* {@code warned} latch can flip, and only once per lifetime of this {@code Retired}. */
static final class Retired {
@@ -157,5 +318,9 @@ final class ClassLoaderGc {
boolean markWarnedIfNotAlready() {
return warned.compareAndSet(false, true);
}
+
+ boolean warnedAlready() {
+ return warned.get();
+ }
}
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/DSLClassLoaderManager.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/DSLClassLoaderManager.java
index 629fab2f28..eab1930329 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/DSLClassLoaderManager.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/DSLClassLoaderManager.java
@@ -75,9 +75,13 @@ public final class DSLClassLoaderManager {
* young-gen pause cycle so most retired loaders surface as collected within a couple of
* ticks; short enough that a leaked loader's WARN doesn't wait minutes. */
private static final long SWEEP_INTERVAL_SECONDS = 30L;
- /** A retired loader still alive past this threshold is WARN'd as a suspected leak. 5 min —
- * long enough that a slow GC cycle doesn't cry wolf, short enough that an actual leak is
- * surfaced before heap pressure triggers a full GC pause. */
+ /** Settle window for the leak WARN: a retired loader is flagged only when a
+ * class-unloading GC cycle is confirmed to have completed at least this long after the
+ * retirement and the loader survived it (see {@link ClassLoaderGc#leakSuspects}).
+ * Wall-clock age alone is never a leak signal — an idle heap may not run a
+ * class-unloading-capable collection for hours, and warning on age produced false
+ * alarms after every hot update on quiet OAPs. 5 min keeps transient holders (an apply
+ * call chain still carrying the displaced bundle on a request thread) out of the WARN. */
private static final long STALE_LOADER_WARN_THRESHOLD_MS = 5L * 60L * 1000L;
private final Map<Key, RuleClassLoader> active = new ConcurrentHashMap<>();
@@ -184,9 +188,10 @@ public final class DSLClassLoaderManager {
}
/**
- * Diagnostic — number of retired loaders the JVM has not yet collected. Steady-state
- * elevated reading is the leak signal the sweeper WARNs on, surfaced here so operators
- * can graph it independently.
+ * Diagnostic — number of retired loaders the JVM has not yet collected. A reading that
+ * stays elevated across class-unloading GC cycles is the leak signal (the sweeper WARNs
+ * on exactly that evidence); an elevated reading on an idle heap merely means no such
+ * cycle has run yet. Surfaced so operators can graph it independently.
*/
public int pendingCount() {
return graveyard.pending().size();
@@ -233,15 +238,37 @@ public final class DSLClassLoaderManager {
log.debug("dsl-classloader-gc: {} loader(s) confirmed collected", collected.size());
}
final long nowMs = System.currentTimeMillis();
- for (final ClassLoaderGc.Retired r : graveyard.pending()) {
- final long ageMs = nowMs - r.retiredAtMs();
- if (ageMs > STALE_LOADER_WARN_THRESHOLD_MS && r.markWarnedIfNotAlready()) {
- log.warn("rule loader leak suspected: {}:{}/{} hash={} pending {} ms "
- + "(threshold {}). Check for lingering handler registrations or "
- + "samples buffered in DataCarrier partitions.",
+ final boolean evidenceBacked = graveyard.unloadProbeUsable();
+ for (final ClassLoaderGc.Retired r : graveyard.leakSuspects(STALE_LOADER_WARN_THRESHOLD_MS)) {
+ if (evidenceBacked) {
+ log.warn("rule loader leak: {}:{}/{} hash={} survived a class-unloading GC cycle "
+ + "and is still strongly referenced {} ms after retirement. This is not GC "
+ + "lag — take a heap dump and inspect the GC-root path of this "
+ + "RuleClassLoader. Common holders: lingering handler registrations, "
+ + "samples buffered in DataCarrier partitions, open DSL debug sessions.",
r.kind() == Kind.BUNDLED ? "bundled" : "runtime-rule",
- r.catalog().getWireName(), r.rule(), r.contentHashShort(), ageMs,
- STALE_LOADER_WARN_THRESHOLD_MS);
+ r.catalog().getWireName(), r.rule(), r.contentHashShort(),
+ nowMs - r.retiredAtMs());
+ } else {
+ log.warn("rule loader leak suspected: {}:{}/{} hash={} still uncollected {} ms "
+ + "after retirement (threshold {}). The unload probe is unavailable, so "
+ + "this is a wall-clock heuristic — it may be GC inactivity rather than "
+ + "a leak. Force a full GC or take a heap dump to confirm.",
+ r.kind() == Kind.BUNDLED ? "bundled" : "runtime-rule",
+ r.catalog().getWireName(), r.rule(), r.contentHashShort(),
+ nowMs - r.retiredAtMs(), STALE_LOADER_WARN_THRESHOLD_MS);
+ }
+ }
+ if (log.isDebugEnabled()) {
+ for (final ClassLoaderGc.Retired r : graveyard.pending()) {
+ if (!r.warnedAlready()) {
+ log.debug("rule loader {}:{}/{} hash={} pending {} ms awaiting a "
+ + "class-unloading GC cycle — not a leak signal (an idle heap "
+ + "can defer collection indefinitely)",
+ r.kind() == Kind.BUNDLED ? "bundled" : "runtime-rule",
+ r.catalog().getWireName(), r.rule(), r.contentHashShort(),
+ nowMs - r.retiredAtMs());
+ }
}
}
} catch (final Throwable t) {
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/UnloadProbePayload.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/UnloadProbePayload.java
new file mode 100644
index 0000000000..0b10381c1b
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/UnloadProbePayload.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.classloader;
+
+/**
+ * Bytecode payload for {@link ClassLoaderGc}'s unload-evidence probe. Never instantiated and
+ * never used as a class — the probe reads this class's {@code .class} resource and defines a
+ * copy of it into a throwaway parent-less classloader, which makes that loader collectible
+ * only by a class-unloading-capable GC cycle (the same collection requirement a retired
+ * {@link RuleClassLoader} has). Keep it empty: the smaller the bytecode, the cheaper each
+ * probe generation.
+ */
+final class UnloadProbePayload {
+ private UnloadProbePayload() {
+ }
+}
diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/classloader/ClassLoaderGcTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/classloader/ClassLoaderGcTest.java
new file mode 100644
index 0000000000..7f6d5f8f01
--- /dev/null
+++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/classloader/ClassLoaderGcTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.classloader;
+
+import java.lang.ref.Reference;
+import java.util.Collection;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Evidence-gating tests for {@link ClassLoaderGc}. A retired loader must never be flagged as
+ * a leak on wall-clock age alone — only after a class-unloading GC cycle demonstrably ran
+ * past the settle window and the loader survived it. Each test uses its own {@code
+ * ClassLoaderGc} instance, so nothing here touches the {@link DSLClassLoaderManager}
+ * singleton's graveyard.
+ *
+ * <p>Deliberately no real-GC test here: asserting that the unload probe is collected by
+ * {@code System.gc()} depends on collector behavior (G1/ZGC/Serial timing, {@code
+ * -XX:+DisableExplicitGC}) and would be a flake source in CI. That physical behavior was
+ * verified manually across collectors when the probe was introduced; these tests pin the
+ * gating logic with injected evidence instead, which is collector-independent.
+ */
+class ClassLoaderGcTest {
+
+ private RuleClassLoader newLoader(final String rule) {
+ return new RuleClassLoader(DSLClassLoaderManager.Kind.RUNTIME, Catalog.LAL, rule,
+ "hash-" + rule, ClassLoaderGcTest.class.getClassLoader());
+ }
+
+ @Test
+ void noSuspectWithoutUnloadEvidence() {
+ final ClassLoaderGc gc = new ClassLoaderGc();
+ final RuleClassLoader pinned = newLoader("no-evidence");
+ gc.retire(pinned);
+ // Even with a zero settle window and any wall-clock age, no completed class-unloading
+ // cycle has been observed, so nothing may be flagged — this is the exact false-alarm
+ // case (idle heap after a hot update).
+ assertTrue(gc.leakSuspects(0).isEmpty(),
+ "no class-unloading GC cycle observed — pending must not be treated as a leak");
+ // JIT liveness analysis may end an object's reachability before its local variable
+ // goes out of scope; the fence guarantees the loader stays "pinned" through the
+ // assertion above regardless of compiler optimization.
+ Reference.reachabilityFence(pinned);
+ }
+
+ @Test
+ void suspectRequiresEvidencePastSettleWindow() {
+ final ClassLoaderGc gc = new ClassLoaderGc();
+ final RuleClassLoader pinned = newLoader("evidence-gated");
+ gc.retire(pinned);
+ // Derive timestamps from the entry itself so the test never reads the wall clock —
+ // an NTP step between retire() and here would otherwise skew the window math.
+ final long retiredAtMs = gc.pending().iterator().next().retiredAtMs();
+
+ // A cycle that completed before the settle window elapsed doesn't qualify: transient
+ // holders (the apply call chain) may legitimately have pinned the loader through it.
+ gc.recordUnloadEvidence(retiredAtMs + 59_999L);
+ assertTrue(gc.leakSuspects(60_000).isEmpty(),
+ "evidence must postdate retirement by the full settle window");
+
+ // A cycle confirmed at retirement + settle flags the entry — exactly once.
+ gc.recordUnloadEvidence(retiredAtMs + 60_000L);
+ final Collection<ClassLoaderGc.Retired> suspects = gc.leakSuspects(60_000);
+ assertEquals(1, suspects.size());
+ assertEquals("evidence-gated", suspects.iterator().next().rule());
+ assertTrue(gc.leakSuspects(60_000).isEmpty(), "leak WARN latch is one-shot per loader");
+
+ Reference.reachabilityFence(pinned);
+ }
+
+ @Test
+ void recordUnloadEvidenceKeepsMaximum() {
+ final ClassLoaderGc gc = new ClassLoaderGc();
+ gc.recordUnloadEvidence(100L);
+ gc.recordUnloadEvidence(50L);
+ assertEquals(100L, gc.unloadEvidenceUpToMs(), "older evidence must not regress the watermark");
+ }
+}