You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maka.apache.org by as...@apache.org on 2026/08/29 17:34:56 UTC
(maka) branch perf/streaming-markdown-preserve-math-source created (now 860892cfd)
This is an automated email from the ASF dual-hosted git repository.
Astro-Han pushed a change to branch perf/streaming-markdown-preserve-math-source
in repository https://gitbox.apache.org/repos/asf/maka.git
at 860892cfd perf(ui): reuse incremental Markdown parsing for math
This branch includes the following new commits:
new 860892cfd perf(ui): reuse incremental Markdown parsing for math
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org
(maka) 01/01: perf(ui): reuse incremental Markdown parsing for math
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Astro-Han pushed a commit to branch perf/streaming-markdown-preserve-math-source
in repository https://gitbox.apache.org/repos/asf/maka.git
commit 860892cfd24ba9a873f35673f204a7bed849536d
Author: AstroHan <le...@outlook.com>
AuthorDate: Sun Aug 30 01:34:15 2026 +0800
perf(ui): reuse incremental Markdown parsing for math
Remove the host-side full-source math tokenization pass and preserve plugin-owned math delimiters inside Astryx's existing incremental parser. This keeps settled block identity while reparsing only the live tail.
Generated-by: Codex
---
packages/ui/src/__tests__/markdown-body.test.ts | 46 +++---
packages/ui/src/markdown-body.tsx | 10 +-
packages/ui/src/markdown-math.tsx | 207 ++++--------------------
patches/@astryxdesign+core+0.5.0.patch | 109 +++++++++++--
patches/README.md | 3 +
5 files changed, 166 insertions(+), 209 deletions(-)
diff --git a/packages/ui/src/__tests__/markdown-body.test.ts b/packages/ui/src/__tests__/markdown-body.test.ts
index c38ec386c..a299bc7db 100644
--- a/packages/ui/src/__tests__/markdown-body.test.ts
+++ b/packages/ui/src/__tests__/markdown-body.test.ts
@@ -21,6 +21,10 @@ import { strict as assert } from 'node:assert';
import { createElement } from 'react';
import { renderToStaticMarkup } from 'react-dom/server';
import { it } from 'node:test';
+import {
+ Markdown as AstryxMarkdown,
+ type MarkdownInlinePlugin,
+} from '@astryxdesign/core/Markdown';
import {
applyMermaidRenderBudget,
MarkdownBody,
@@ -37,6 +41,25 @@ import {
MAX_MERMAID_SOURCE_LENGTH,
} from '../mermaid-diagram.js';
+it('lets inline plugins preserve source syntax consumed by Markdown parsing', () => {
+ const plugin = {
+ pattern: /\\\(([^\n]+?)\\\)/g,
+ preserveSource: true,
+ render: (match, key) => createElement('span', {
+ key,
+ 'data-formula': match[1],
+ }),
+ } satisfies MarkdownInlinePlugin;
+
+ const markup = renderToStaticMarkup(createElement(AstryxMarkdown, {
+ inlinePlugins: [plugin],
+ children: 'Set \\( x + 1 \\) in prose and keep `\\( code \\)` literal.',
+ }));
+
+ assert.match(markup, /data-formula=" x \+ 1 "/);
+ assert.match(markup, /<code[^>]*>\\\( code \\\)<\/code>/);
+});
+
it('keeps raw HTML inert instead of expanding the Markdown trust surface', () => {
const markup = renderToStaticMarkup(createElement(MarkdownBody, {
text: '<details open><summary>Click</summary>payload</details>',
@@ -99,21 +122,6 @@ it('does not treat shell variables, currency, or inline code as dollar-delimited
assert.match(markup, /class="katex"/);
});
-it('keeps raw and malformed internal-token lookalikes literal', () => {
- for (const token of [
- '\uE000MAKAMATHIFFFFFFEND\uE001',
- '\uE000MAKAMATH:0:0:\uE001',
- '\uE000MAKAMATH:999:not-a-token:\uE001',
- ]) {
- const markup = renderToStaticMarkup(createElement(MarkdownBody, {
- text: `Before ${token} after with \\( x + 1 \\).`,
- }));
-
- assert.match(markup, new RegExp(token));
- assert.equal((markup.match(/class="maka-math maka-math-inline"/g) ?? []).length, 1);
- }
-});
-
it('renders multiline display math outside code for both supported delimiters', () => {
for (const [text, mathNode] of [
[['Before', '', '$$', 'E = mc^2', '$$', '', 'After'].join('\n'), '<msup>'],
@@ -409,15 +417,13 @@ it('shows only the restored prefix on its first streaming render', () => {
assert.doesNotMatch(markup, /new delta/);
});
-it('uses the same protected math registry for live and settled streaming text', () => {
- const rawToken = '\uE000MAKAMATH:0:0:\uE001';
+it('renders settled math while keeping the live tail behind the display cursor', () => {
const markup = renderToStaticMarkup(createElement(MarkdownBody, {
- text: `Stable ${rawToken} \\( x + 1 \\) with a new delta`,
+ text: 'Stable \\( x + 1 \\) with a new delta',
streaming: true,
- settledText: `Stable ${rawToken} \\( x + 1 \\)`,
+ settledText: 'Stable \\( x + 1 \\)',
}));
- assert.match(markup, new RegExp(rawToken));
assert.match(markup, /class="maka-math maka-math-inline"/);
assert.match(markup, /class="katex"/);
assert.doesNotMatch(markup, /new delta/);
diff --git a/packages/ui/src/markdown-body.tsx b/packages/ui/src/markdown-body.tsx
index 1687ae12b..20b795e9b 100644
--- a/packages/ui/src/markdown-body.tsx
+++ b/packages/ui/src/markdown-body.tsx
@@ -46,7 +46,7 @@ import { MakaUriContext } from './markdown.js';
import { useUiLocale } from './locale-context.js';
import { getSharedUiCopy } from './shared-ui-copy.js';
import { MermaidDiagram } from './mermaid-diagram.js';
-import { prepareMarkdownMath } from './markdown-math.js';
+import { MARKDOWN_MATH_PLUGINS } from './markdown-math.js';
import { parseAttachmentResourceRef } from '@maka/core/attachments';
import { useAttachmentImageSource } from './attachment-image.js';
@@ -132,9 +132,7 @@ export function MarkdownBody(props: {
settledText?: string;
density?: 'default' | 'compact';
}) {
- const prepared = prepareMarkdownMath(props.text, props.settledText);
- const safeText = prepared.text;
- const budgetedText = props.streaming ? safeText : applyMermaidRenderBudget(safeText);
+ const budgetedText = props.streaming ? props.text : applyMermaidRenderBudget(props.text);
const density = props.density ?? 'default';
const components = props.streaming
? density === 'compact'
@@ -175,9 +173,9 @@ export function MarkdownBody(props: {
// the one combination neither half of the argument asks for.
density={density}
components={components}
- inlinePlugins={[prepared.plugin]}
+ inlinePlugins={MARKDOWN_MATH_PLUGINS}
isStreaming={props.streaming}
- settledText={prepared.settledText}
+ settledText={props.settledText}
>
{budgetedText}
</AstryxMarkdown>
diff --git a/packages/ui/src/markdown-math.tsx b/packages/ui/src/markdown-math.tsx
index eccd8cbb9..b8dd86471 100644
--- a/packages/ui/src/markdown-math.tsx
+++ b/packages/ui/src/markdown-math.tsx
@@ -20,177 +20,38 @@
import katex from 'katex';
import type { MarkdownInlinePlugin } from '@astryxdesign/core/Markdown';
-export interface PreparedMarkdownMath {
- text: string;
- settledText?: string;
- plugin: MarkdownInlinePlugin;
-}
-
-interface MathTokenValue {
- formula: string;
- displayMode: boolean;
-}
-
-export function prepareMarkdownMath(
- source: string,
- settledSource?: string,
-): PreparedMarkdownMath {
- const registry = createMathTokenRegistry([source, settledSource]);
- return {
- text: protectMathOutsideCode(source, registry.register),
- ...(settledSource === undefined
- ? {}
- : { settledText: protectMathOutsideCode(settledSource, registry.register) }),
- plugin: {
- pattern: registry.pattern,
- render: (match, key) => {
- const value = registry.values.get(match[0]);
- if (!value) return match[0];
- const html = katex.renderToString(value.formula, {
- displayMode: value.displayMode,
- output: 'htmlAndMathml',
- strict: 'warn',
- throwOnError: false,
- trust: false,
- });
- return (
- <span
- key={key}
- className={
- value.displayMode ? 'maka-math maka-math-display' : 'maka-math maka-math-inline'
- }
- dangerouslySetInnerHTML={{ __html: html }}
- />
- );
- },
- },
- };
-}
-
-function createMathTokenRegistry(sources: Array<string | undefined>): {
- pattern: RegExp;
- register: (formula: string, displayMode: boolean) => string;
- values: Map<string, MathTokenValue>;
-} {
- let namespaceIndex = 0;
- let namespace = '';
- do {
- namespace = `\uE000MAKAMATH:${namespaceIndex}:`;
- namespaceIndex += 1;
- } while (sources.some((source) => source?.includes(namespace)));
-
- const tokenEnd = ':\uE001';
- const values = new Map<string, MathTokenValue>();
- const tokensByValue = new Map<string, string>();
- let nextTokenId = 0;
- return {
- pattern: new RegExp(`${escapeRegExp(namespace)}\\d+${escapeRegExp(tokenEnd)}`, 'g'),
- register: (formula, displayMode) => {
- const valueKey = JSON.stringify([displayMode, formula]);
- const existingToken = tokensByValue.get(valueKey);
- if (existingToken) return existingToken;
-
- const token = `${namespace}${nextTokenId}${tokenEnd}`;
- nextTokenId += 1;
- values.set(token, { formula, displayMode });
- tokensByValue.set(valueKey, token);
- return token;
- },
- values,
- };
-}
-
-function protectMathOutsideCode(
- source: string,
- register: (formula: string, displayMode: boolean) => string,
-): string {
- const lines = source.split('\n');
- let fence: { character: string; length: number } | undefined;
- let proseLines: string[] = [];
- const protectedParts: string[] = [];
-
- const flushProse = () => {
- if (proseLines.length === 0) return;
- protectedParts.push(protectMathInProse(proseLines.join('\n'), register));
- proseLines = [];
- };
-
- for (const line of lines) {
- const opening = /^( {0,3})(`{3,}|~{3,})/.exec(line);
- if (opening) {
- flushProse();
- const marker = opening[2] ?? '';
- const character = marker[0] ?? '';
- if (!fence) {
- fence = { character, length: marker.length };
- } else if (character === fence.character && marker.length >= fence.length) {
- fence = undefined;
- }
- protectedParts.push(line);
- } else if (fence) {
- protectedParts.push(line);
- } else {
- proseLines.push(line);
- }
- }
- flushProse();
-
- return protectedParts.join('\n');
-}
-
-function protectMathInProse(
- source: string,
- register: (formula: string, displayMode: boolean) => string,
-): string {
- let output = '';
- let index = 0;
-
- while (index < source.length) {
- if (source[index] === '`') {
- const run = /^`+/.exec(source.slice(index))?.[0] ?? '`';
- const close = source.indexOf(run, index + run.length);
- if (close >= 0) {
- output += source.slice(index, close + run.length);
- index = close + run.length;
- continue;
- }
- }
-
- const delimited =
- readDelimitedMath(source, index, '\\(', '\\)', false, false)
- ?? readDelimitedMath(source, index, '\\[', '\\]', true, true)
- ?? readDelimitedMath(source, index, '$$', '$$', true, true);
- if (delimited) {
- output += register(delimited.formula, delimited.displayMode);
- index = delimited.end;
- continue;
- }
-
- output += source[index];
- index += 1;
- }
-
- return output;
-}
-
-function readDelimitedMath(
- line: string,
- index: number,
- opening: string,
- closing: string,
- displayMode: boolean,
- allowNewlines: boolean,
-): { formula: string; displayMode: boolean; end: number } | undefined {
- if (!line.startsWith(opening, index)) return undefined;
- const contentStart = index + opening.length;
- const close = line.indexOf(closing, contentStart);
- if (close < 0) return undefined;
- if (!allowNewlines && line.slice(contentStart, close).includes('\n')) return undefined;
- const formula = line.slice(contentStart, close).trim();
- if (!formula) return undefined;
- return { formula, displayMode, end: close + closing.length };
-}
-
-function escapeRegExp(value: string): string {
- return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+const markdownMathPlugin = {
+ pattern:
+ /\\\((?<inline>[^\n]+?)\\\)|\\\[(?<bracket>[\s\S]+?)\\\]|\$\$(?<dollar>[\s\S]+?)\$\$/g,
+ preserveSource: true,
+ getEndIndex: (_text, match) => {
+ const formula = readFormula(match);
+ return formula === '' ? false : (match.index ?? 0) + match[0].length;
+ },
+ render: (match, key) => {
+ const formula = readFormula(match);
+ const displayMode = match.groups?.inline === undefined;
+ const html = katex.renderToString(formula, {
+ displayMode,
+ output: 'htmlAndMathml',
+ strict: 'warn',
+ throwOnError: false,
+ trust: false,
+ });
+ return (
+ <span
+ key={key}
+ className={
+ displayMode ? 'maka-math maka-math-display' : 'maka-math maka-math-inline'
+ }
+ dangerouslySetInnerHTML={{ __html: html }}
+ />
+ );
+ },
+} satisfies MarkdownInlinePlugin;
+
+export const MARKDOWN_MATH_PLUGINS = [markdownMathPlugin] satisfies MarkdownInlinePlugin[];
+
+function readFormula(match: RegExpMatchArray): string {
+ return (match.groups?.inline ?? match.groups?.bracket ?? match.groups?.dollar ?? '').trim();
}
diff --git a/patches/@astryxdesign+core+0.5.0.patch b/patches/@astryxdesign+core+0.5.0.patch
index 0b7127ec3..723f265b0 100644
--- a/patches/@astryxdesign+core+0.5.0.patch
+++ b/patches/@astryxdesign+core+0.5.0.patch
@@ -154,10 +154,19 @@ index 08b5f7d..94c623b 100644
...(isOrdered && start != null && start !== 1 ? {
start
diff --git a/node_modules/@astryxdesign/core/dist/Markdown/Markdown.d.ts b/node_modules/@astryxdesign/core/dist/Markdown/Markdown.d.ts
-index d9ad714..2b9c461 100644
+index d9ad714..5381e71 100644
--- a/node_modules/@astryxdesign/core/dist/Markdown/Markdown.d.ts
+++ b/node_modules/@astryxdesign/core/dist/Markdown/Markdown.d.ts
-@@ -90,6 +90,8 @@ export interface MarkdownProps extends BaseProps<HTMLElement> {
+@@ -15,6 +15,8 @@ import type { CitationSource } from '../Citation/Citation';
+ export interface MarkdownInlinePlugin {
+ /** Regex with global flag. Matched against text nodes only. */
+ pattern: RegExp;
++ /** Preserve matching source syntax before Markdown parses prose text. */
++ preserveSource?: boolean;
+ /**
+ * Optional: refine the match boundary after the regex hits.
+ * Return the end index, or false to reject the match.
+@@ -90,6 +92,8 @@ export interface MarkdownProps extends BaseProps<HTMLElement> {
*/
headingLevelStart?: 1 | 2 | 3 | 4 | 5 | 6;
isStreaming?: boolean;
@@ -166,7 +175,7 @@ index d9ad714..2b9c461 100644
onLinkClick?: (href: string, event: React.MouseEvent<HTMLAnchorElement>) => void | false;
/**
* Citation sources keyed by ID. When provided, `[id]` and `【id】` markers
-@@ -155,7 +157,7 @@ export interface MarkdownProps extends BaseProps<HTMLElement> {
+@@ -155,7 +159,7 @@ export interface MarkdownProps extends BaseProps<HTMLElement> {
* </Markdown>
* ```
*/
@@ -176,7 +185,7 @@ index d9ad714..2b9c461 100644
var displayName: string;
}
diff --git a/node_modules/@astryxdesign/core/dist/Markdown/Markdown.js b/node_modules/@astryxdesign/core/dist/Markdown/Markdown.js
-index 0fd8329..7e4ec57 100644
+index 0fd8329..d3d27f1 100644
--- a/node_modules/@astryxdesign/core/dist/Markdown/Markdown.js
+++ b/node_modules/@astryxdesign/core/dist/Markdown/Markdown.js
@@ -8,7 +8,7 @@
@@ -196,7 +205,15 @@ index 0fd8329..7e4ec57 100644
onLinkClick,
sources,
citationStyle = 'label',
-@@ -1072,7 +1073,9 @@ export function Markdown({
+@@ -1067,20 +1068,16 @@ export function Markdown({
+ const sourceIds = useMemo(() => sources ? new Set(Object.keys(sources)) : undefined, [sources]);
+ const parseOptions = useMemo(() => ({
+ sourceIds,
+- autolink
+- }), [sourceIds, autolink]);
++ autolink,
++ preserveSourcePatterns: inlinePlugins?.filter(plugin => plugin.preserveSource).map(plugin => new RegExp(plugin.pattern.source, `${plugin.pattern.flags.replace(/[gy]/g, '')}y`))
++ }), [sourceIds, autolink, inlinePlugins]);
// Smooth bursty streamed chunks into a steady character-by-character reveal.
// When not streaming, the hook returns children unchanged (no-op).
@@ -205,9 +222,17 @@ index 0fd8329..7e4ec57 100644
+ settledText
+ });
const incrementalStateRef = useRef(createIncrementalState());
- // Reset incremental cache when the autolink option toggles — cached
- // settled blocks were parsed with the previous setting.
-@@ -1139,17 +1142,25 @@ export function Markdown({
+- // Reset incremental cache when the autolink option toggles — cached
+- // settled blocks were parsed with the previous setting.
+- const prevAutolinkRef = useRef(autolink);
+- if (prevAutolinkRef.current !== autolink) {
+- incrementalStateRef.current = createIncrementalState();
+- prevAutolinkRef.current = autolink;
+- }
+ const blocks = useMemo(() => {
+ if (display === 'inline') {
+ return [];
+@@ -1139,17 +1136,25 @@ export function Markdown({
const tickMs = tick != null ? Math.max(4, Math.round(tick / 10)) : 50;
return Math.min(Math.ceil(duration / tickMs), 12);
}, [token]);
@@ -244,7 +269,7 @@ index 0fd8329..7e4ec57 100644
const cursor = {
offset: 0,
boundaries,
-@@ -1174,9 +1185,6 @@ export function Markdown({
+@@ -1174,9 +1179,6 @@ export function Markdown({
children: inlineNodes.map((node, i) => renderInline(node, i, onLinkClick, cursor, citationCtx, LinkComponent, inlinePlugins, components))
});
@@ -254,7 +279,7 @@ index 0fd8329..7e4ec57 100644
return renderedInline;
}
const rendered = /*#__PURE__*/_jsx("div", {
-@@ -1189,11 +1197,6 @@ export function Markdown({
+@@ -1189,11 +1191,6 @@ export function Markdown({
children: blocks.map((block, i) => renderBlock(block, i, blocks.length, density, headingLevelStart, onLinkClick, cursor, citationCtx, contentWidth ? typeof contentWidth === 'number' ? `${contentWidth}px` : contentWidth : null, contentAlign, LinkComponent, inlinePlugins, components, t, headingIdMap))
});
@@ -267,6 +292,70 @@ index 0fd8329..7e4ec57 100644
}
Markdown.displayName = 'Markdown';
\ No newline at end of file
+diff --git a/node_modules/@astryxdesign/core/dist/Markdown/parser.js b/node_modules/@astryxdesign/core/dist/Markdown/parser.js
+index cc9a01a..54a7564 100644
+--- a/node_modules/@astryxdesign/core/dist/Markdown/parser.js
++++ b/node_modules/@astryxdesign/core/dist/Markdown/parser.js
+@@ -58,6 +58,7 @@ function resolveOptions(arg) {
+ return {
+ sourceIds: opts.sourceIds,
+ autolink: opts.autolink,
+- sourceRanges: opts.sourceRanges
++ sourceRanges: opts.sourceRanges,
++ preserveSourcePatterns: opts.preserveSourcePatterns
+ };
+ }
+@@ -416,6 +417,29 @@ function parseInlineImpl(text, opts) {
+ const nodes = [];
+ let i = 0;
+ while (i < text.length) {
++ // Preserve plugin-owned source syntax before Markdown consumes escapes or
++ // delimiters. Inline code is still parsed as one code node, so plugins
++ // cannot match inside it.
++ if (text[i] !== '`' && opts.preserveSourcePatterns) {
++ let preserved;
++ for (const pattern of opts.preserveSourcePatterns) {
++ pattern.lastIndex = i;
++ const match = pattern.exec(text);
++ if (match && match[0].length > 0) {
++ preserved = match[0];
++ break;
++ }
++ }
++ if (preserved !== undefined) {
++ nodes.push({
++ type: 'text',
++ content: preserved
++ });
++ i += preserved.length;
++ continue;
++ }
++ }
++
+ // --- Escape ---
+ if (text[i] === '\\' && i + 1 < text.length) {
+ nodes.push({
+@@ -1693,15 +1717,16 @@ function mergeSettledBlocks(prev, delta) {
+ }
+ export function parseMarkdownIncremental(input, state, arg) {
+ const opts = resolveOptions(arg);
+- // Invalidate cache when the autolink or sourceRanges option flips — cached
+- // settled blocks were parsed with the previous setting and would otherwise
+- // be reused unchanged.
+- if (state.autolink !== opts.autolink || Boolean(state.sourceRanges) !== Boolean(opts.sourceRanges)) {
++ const preserveSourcePatternsKey = opts.preserveSourcePatterns?.map(pattern => `${pattern.source}\u0000${pattern.flags}`).join('\u0001') ?? '';
++ // Invalidate cache when a parse option flips — cached settled blocks were
++ // parsed with the previous behavior and would otherwise be reused unchanged.
++ if (state.autolink !== opts.autolink || Boolean(state.sourceRanges) !== Boolean(opts.sourceRanges) || state.preserveSourcePatternsKey !== preserveSourcePatternsKey) {
+ state.prevInput = '';
+ state.settledText = '';
+ state.settledBlocks = [];
+ state.settledUpTo = 0;
+ state.autolink = opts.autolink;
++ state.preserveSourcePatternsKey = preserveSourcePatternsKey;
+ state.sourceRanges = opts.sourceRanges;
+ }
+ if (input === '') {
diff --git a/node_modules/@astryxdesign/core/dist/SideNav/SideNavItem.d.ts b/node_modules/@astryxdesign/core/dist/SideNav/SideNavItem.d.ts
index e7b8e00..fad2e72 100644
--- a/node_modules/@astryxdesign/core/dist/SideNav/SideNavItem.d.ts
diff --git a/patches/README.md b/patches/README.md
index 6ed332c9a..2edbb61fe 100644
--- a/patches/README.md
+++ b/patches/README.md
@@ -76,5 +76,8 @@ Streaming text and Markdown expose an explicit `settledText` seam so the
renderer can verify and advance the exact prefix already presented without
replaying it. The default remains progressive for a genuinely new stream, and
rewritten or later text still reveals and fades from a parsed-visible boundary.
+Markdown inline plugins can also preserve matching source syntax inside the
+existing incremental parse, so hosts do not need a full-source preprocessing
+pass for syntax such as math delimiters.
Delete each hunk when the corresponding behavior ships in Astryx.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org