(maka) branch main updated: perf(runtime): bound Read line-index memory to the page (#5461)

[as...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

Astro-Han pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/maka.git


The following commit(s) were added to refs/heads/main by this push:
     new 20ed4770e6 perf(runtime): bound Read line-index memory to the page (#5461)
20ed4770e6 is described below

commit 20ed4770e69356c254de9b0a41c0a1031dd6d0ab
Author: seekskyworld <14...@users.noreply.github.com>
AuthorDate: Tue Oct 13 08:47:37 2026 +0800

    perf(runtime): bound Read line-index memory to the page (#5461)
    
    A bounded Read page currently allocates one newline offset for every source line, even when requesting only 20 lines. Count total lines with scalar state and retain offsets only within the possible response window. Exact range, continuation-digest, Unicode and response-size semantics remain covered by the existing pager tests.
    
    Generated-by: OpenAI Codex
---
 packages/runtime/src/__tests__/read-page.test.ts | 46 ++++++++++++++++++++++++
 packages/runtime/src/read-page.ts                | 40 +++++++++++++++------
 2 files changed, 76 insertions(+), 10 deletions(-)

diff --git a/packages/runtime/src/__tests__/read-page.test.ts b/packages/runtime/src/__tests__/read-page.test.ts
index a07529e7b9..ef38299d67 100644
--- a/packages/runtime/src/__tests__/read-page.test.ts
+++ b/packages/runtime/src/__tests__/read-page.test.ts
@@ -18,6 +18,8 @@
  */
 
 import assert from 'node:assert/strict';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
 import { test } from 'node:test';
 import {
   readPage,
@@ -175,3 +177,47 @@ test('archive Read preserves fields beside structured text across pages', () =>
     assert.deepEqual(JSON.parse(recovered), value);
   }
 });
+
+test('small line ranges do not need a whole-file line index', async () => {
+  // 固定堆预算验证实际资源边界，不依赖耗时阈值或具体索引实现。
+  const { stdout } = await promisify(execFile)(
+    process.execPath,
+    [
+      '--max-old-space-size=64',
+      '--input-type=module',
+      '--eval',
+      `
+        import assert from 'node:assert/strict';
+        import { readPage } from ${JSON.stringify(new URL('../read-page.js', import.meta.url).href)};
+        const content = 'x\\n'.repeat(8_000_000);
+        const page = readPage(content, { path: 'log.txt', offset: 2_000_000, limit: 20 });
+        assert.equal(page.totalLines, 8_000_001);
+        assert.equal(page.offset, 2_000_000);
+        assert.equal(page.returnedLines, 20);
+        assert.equal(page.content, 'x\\n'.repeat(19) + 'x');
+        assert.equal(page.next, null);
+        console.log('bounded read passed');
+      `,
+    ],
+    { timeout: 30_000 },
+  );
+  assert.equal(stdout.trim(), 'bounded read passed');
+});
+
+test('line ranges preserve empty lines, trailing newlines and offsets beyond EOF', () => {
+  for (const content of ['', '\n', '\n\n', 'a\nb', 'a\nb\n', 'a\r\n😀\n\nlast']) {
+    const lines = content.split('\n');
+    for (let offset = 0; offset <= lines.length + 1; offset++) {
+      for (const limit of [undefined, 1, 2, 100]) {
+        const selected = lines.slice(offset, limit === undefined ? undefined : offset + limit);
+        assert.deepEqual(readPage(content, { path: 'file.txt', offset, limit }), {
+          content: selected.join('\n'),
+          offset,
+          returnedLines: selected.length,
+          totalLines: lines.length,
+          next: null,
+        });
+      }
+    }
+  }
+});
diff --git a/packages/runtime/src/read-page.ts b/packages/runtime/src/read-page.ts
index 11a0c50dd7..79e61ea05c 100644
--- a/packages/runtime/src/read-page.ts
+++ b/packages/runtime/src/read-page.ts
@@ -121,10 +121,34 @@ export function readPage(
       'Invalid Read continuation position. Copy the complete next object from the preceding result.',
     );
   }
-  const starts = [0];
-  for (let at = content.indexOf('\n'); at >= 0; at = content.indexOf('\n', at + 1))
+  // Count all lines, but retain offsets only for this response window. A UTF-8
+  // byte budget also bounds its UTF-16 length, including multibyte content.
+  let totalLines = 1;
+  let offset = resolved.position === undefined ? (input.offset ?? 0) : 0;
+  let start = resolved.position ?? (offset === 0 ? 0 : content.length);
+  let lineStart = 0;
+  let end = content.length;
+  for (let at = content.indexOf('\n'); at >= 0; at = content.indexOf('\n', at + 1)) {
+    const line = totalLines++;
+    if (resolved.position !== undefined && at < resolved.position) {
+      offset = line;
+      lineStart = at + 1;
+    } else if (resolved.position === undefined && line === offset) {
+      start = at + 1;
+      lineStart = start;
+    }
+    if (input.limit !== undefined && line === offset + input.limit && at >= start) end = at;
+  }
+  const lastLine = Math.min(totalLines, offset + (input.limit ?? totalLines));
+  const starts = [lineStart];
+  const indexEnd = Math.min(end, start + maxBytes);
+  for (
+    let at = content.indexOf('\n', start);
+    at >= 0 && at < indexEnd;
+    at = content.indexOf('\n', at + 1)
+  ) {
     starts.push(at + 1);
-  const totalLines = starts.length;
+  }
   const lineAt = (position: number): number => {
     let low = 0;
     let high = starts.length - 1;
@@ -133,17 +157,13 @@ export function readPage(
       if (starts[middle]! <= position) low = middle;
       else high = middle - 1;
     }
-    return low;
+    return offset + low;
   };
-  const offset = resolved.position === undefined ? (input.offset ?? 0) : lineAt(resolved.position);
-  const start = resolved.position ?? starts[Math.min(offset, totalLines)] ?? content.length;
-  const lastLine = Math.min(totalLines, offset + (input.limit ?? totalLines));
-  const end = lastLine < totalLines ? starts[lastLine]! - 1 : content.length;
   let digest = resolved.digest;
   const makePage = (stop: number): ReadPage => {
     const complete = stop >= end;
     const nextLine = Math.max(offset, lineAt(stop));
-    const boundary = stop === (starts[nextLine + 1] ?? content.length + 1) - 1;
+    const boundary = content[stop] === '\n';
     const partialLine = !complete && !boundary;
     const returnedLines =
       offset >= totalLines
@@ -165,7 +185,7 @@ export function readPage(
       offset,
       returnedLines,
       totalLines,
-      ...(partialLine || (offset < totalLines && start > starts[offset]!)
+      ...(partialLine || (offset < totalLines && start > lineStart)
         ? { partialLine: true as const }
         : {}),
       next,


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org