(maka) branch main updated: fix(core): scan fully for a recall term with an unpaired surrogate (#5422)

[as...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

Astro-Han pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/maka.git


The following commit(s) were added to refs/heads/main by this push:
     new 882d0402c1 fix(core): scan fully for a recall term with an unpaired surrogate (#5422)
882d0402c1 is described below

commit 882d0402c1a8366aa20a45dd69f2e245d58abc0c
Author: Totoro <ms...@gmail.com>
AuthorDate: Sun Oct 11 20:29:36 2026 +0800

    fix(core): scan fully for a recall term with an unpaired surrogate (#5422)
    
    The candidate source searches the serialized transcript, where
    JSON.stringify writes a lone surrogate as a \u escape, and SQLite binds
    a lone-surrogate term as U+FFFD. A Recall term containing one therefore
    got an empty candidate list, which collectHits treats as authoritative,
    and a passage the verifier would match was silently dropped.
    
    Treat such a term like the other escaped characters and take the full
    scan. A well-formed surrogate pair is one code point under the u flag
    and keeps the candidate path.
    
    Generated-by: Claude Code
    test(core): pin a lone low surrogate in the recall full-scan guard
    
    Also say why a lone surrogate forces a full scan even where it is stored
    literally as half of a pair, name the term in the passage assertion, and
    note in recall-fold that its stable-record predicate relies on recall
    scanning escaped and surrogate terms fully.
    
    Generated-by: Claude Code
---
 packages/core/src/__tests__/recall.test.ts | 44 ++++++++++++++++++++++++++++++
 packages/core/src/recall.ts                | 13 ++++++++-
 packages/storage/src/recall-fold.ts        |  5 ++++
 3 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/packages/core/src/__tests__/recall.test.ts b/packages/core/src/__tests__/recall.test.ts
index 63e97a12e6..10cf2a2a08 100644
--- a/packages/core/src/__tests__/recall.test.ts
+++ b/packages/core/src/__tests__/recall.test.ts
@@ -405,6 +405,50 @@ test('a term the stored form escapes bypasses the candidate source', async () =>
   assert.equal(asked, 0);
 });
 
+test('a term with an unpaired surrogate bypasses the candidate source', async () => {
+  // `JSON.stringify` writes a lone surrogate as a `\ud800` escape, and SQLite
+  // binds one as U+FFFD, so the candidate source cannot vouch for a term
+  // containing one. The parsed text still matches it.
+  const data = corpus([
+    {
+      session: session('s-lone', 'lone surrogate'),
+      messages: [userMessage('m-lone', 't-lone', 'abc\uD800def')],
+    },
+    {
+      session: session('s-emoji', 'emoji'),
+      messages: [userMessage('m-emoji', 't-emoji', 'ship it 🚀 today')],
+    },
+  ]);
+  for (const [term, anchor] of [
+    ['\uD800', 'm-lone'],
+    ['c\uD800d', 'm-lone'],
+    // Either half of the pair in the stored emoji: this double finds it, SQLite
+    // does not.
+    ['\uD83D', 'm-emoji'],
+    ['\uDE80', 'm-emoji'],
+  ] as const) {
+    const result = await runRecall({ terms: [term] }, candidateDeps(data));
+    assert.ok(result.ok);
+    assert.equal(
+      result.scannedFully,
+      true,
+      `${JSON.stringify(term)} must not use the candidate source`,
+    );
+    assert.deepEqual(
+      anchorIds(result.passages),
+      [anchor],
+      `${JSON.stringify(term)} must return its passage`,
+    );
+  }
+
+  // A surrogate pair is one character, stored literally, so the candidate
+  // source can still vouch for it.
+  const paired = await runRecall({ terms: ['🚀'] }, candidateDeps(data));
+  assert.ok(paired.ok);
+  assert.equal(paired.scannedFully, false);
+  assert.deepEqual(anchorIds(paired.passages), ['m-emoji']);
+});
+
 test('a dense tool result does not outrank the answer that explains it', async () => {
   const data = mixedCorpus();
   // `m3` is shaped like a grep result: forty repetitions of every term. BM25's
diff --git a/packages/core/src/recall.ts b/packages/core/src/recall.ts
index 29470f5b1e..30116cf4c8 100644
--- a/packages/core/src/recall.ts
+++ b/packages/core/src/recall.ts
@@ -133,13 +133,24 @@ const JSON_ESCAPED_PRINTABLE_PATTERN = /["\\]/u;
 /** First code point `JSON.stringify` leaves unescaped. */
 const FIRST_UNESCAPED_CODE_POINT = 0x20;
 
+/**
+ * A surrogate without its partner. `JSON.stringify` writes one as a `\u`
+ * escape, and a store binding the term as UTF-8 cannot represent it at all, so
+ * not even a stored pair it is half of is offered. Under the `u` flag a
+ * well-formed pair is one code point, so an emoji does not match.
+ */
+const UNPAIRED_SURROGATE_PATTERN = /\p{Cs}/u;
+
 /**
  * A term containing a character that `JSON.stringify` escapes is stored in a
  * different literal form than it was typed, so a candidate source scanning
- * serialized records would under-select it. Such terms force a full scan.
+ * serialized records would under-select it. A lone surrogate also cannot be
+ * bound as UTF-8, so such a scan misses it even where it is stored literally,
+ * as half of a pair. Such terms force a full scan.
  */
 function hasJsonEscapedCharacter(term: string): boolean {
   if (JSON_ESCAPED_PRINTABLE_PATTERN.test(term)) return true;
+  if (UNPAIRED_SURROGATE_PATTERN.test(term)) return true;
   for (const character of term) {
     if ((character.codePointAt(0) ?? 0) < FIRST_UNESCAPED_CODE_POINT) return true;
   }
diff --git a/packages/storage/src/recall-fold.ts b/packages/storage/src/recall-fold.ts
index a36e7927e1..2be52c26ff 100644
--- a/packages/storage/src/recall-fold.ts
+++ b/packages/storage/src/recall-fold.ts
@@ -31,6 +31,11 @@ import type { DatabaseSync } from 'node:sqlite';
  * that NFC rewrites. On stable records `instr(lower(stored), term)` is exactly
  * recall's predicate, and on unstable ones the record is a candidate anyway,
  * so the scan never under-selects whatever script the transcript is in.
+ *
+ * That holds for terms the serialized record spells the way they were typed.
+ * A term with a character `JSON.stringify` escapes, or a lone surrogate, which
+ * cannot be bound at all, never reaches this scan: recall scans those fully
+ * (`hasJsonEscapedCharacter` in core `recall.ts`).
  */
 export const RECALL_FOLD_UNSTABLE_FUNCTION = 'maka_recall_fold_unstable';
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org