You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maka.apache.org by wy...@apache.org on 2026/10/22 22:06:47 UTC
(maka) 19/29: fix(terminal-shell): match source maps to Markdown lexical whitespace
This is an automated email from the ASF dual-hosted git repository.
M4n5ter pushed a commit to branch rust-preview
in repository https://gitbox.apache.org/repos/asf/maka.git
commit a99b4c7640970ad4f54f11b202589e5781556d8f
Author: M4n5ter <m4...@gmail.com>
AuthorDate: Fri Oct 23 04:45:56 2026 +0800
fix(terminal-shell): match source maps to Markdown lexical whitespace
---
products/maka/terminal-shell/src/markdown.rs | 41 ++++++++++
.../maka/terminal-shell/src/markdown/source.rs | 87 ++++++++++++----------
2 files changed, 89 insertions(+), 39 deletions(-)
diff --git a/products/maka/terminal-shell/src/markdown.rs b/products/maka/terminal-shell/src/markdown.rs
index 274828c1f0..f9a9e35bb7 100644
--- a/products/maka/terminal-shell/src/markdown.rs
+++ b/products/maka/terminal-shell/src/markdown.rs
@@ -623,6 +623,10 @@ mod tests {
);
let invalid = parse_source("& �").unwrap();
assert_eq!(invalid[0].runs[0].text, "& �");
+ for original in ["&", "&"] {
+ let invalid = parse_source(&format!("& {original}")).unwrap();
+ assert_eq!(invalid[0].runs[0].text, format!("& {original}"));
+ }
assert_eq!(blocks[0].runs[1].text, "a🙂");
assert_eq!(
blocks[0].runs[1].source,
@@ -634,6 +638,31 @@ mod tests {
kind: MarkdownSourceKind::Literal,
}]
);
+ for (original, expected, ranges) in [
+ ("a \nb", "a\nb", vec![(0, 1, 0, 1), (1, 3, 2, 4)]),
+ ("a\t \nb", "a\nb", vec![(0, 1, 0, 1), (1, 3, 3, 5)]),
+ (
+ "a\n  b",
+ "a\n b",
+ vec![(0, 2, 0, 2), (2, 3, 4, 9), (3, 4, 9, 10)],
+ ),
+ ] {
+ let blocks = parse_source(original).unwrap();
+ let run = &blocks[0].runs[0];
+ assert_eq!(run.text, expected);
+ assert_eq!(
+ run.source
+ .iter()
+ .map(|segment| (
+ segment.start,
+ segment.end,
+ segment.source_start,
+ segment.source_end,
+ ))
+ .collect::<Vec<_>>(),
+ ranges,
+ );
+ }
}
#[test]
@@ -758,6 +787,18 @@ mod tests {
.collect::<Vec<_>>(),
[(0, 4, 4, 8), (4, 6, 12, 14), (6, 9, 18, 21)]
);
+ let nul = parse_source("```\na\0b\n```").unwrap();
+ assert_eq!(nul[0].text, "a\0b");
+ assert_eq!(
+ nul[0].source,
+ [MarkdownSourceSegment {
+ start: 0,
+ end: 3,
+ source_start: 4,
+ source_end: 7,
+ kind: MarkdownSourceKind::Literal,
+ }]
+ );
}
#[test]
diff --git a/products/maka/terminal-shell/src/markdown/source.rs b/products/maka/terminal-shell/src/markdown/source.rs
index 7cfc2574fc..988a5506ae 100644
--- a/products/maka/terminal-shell/src/markdown/source.rs
+++ b/products/maka/terminal-shell/src/markdown/source.rs
@@ -122,13 +122,28 @@ pub(super) fn text(
let base = start(node) + offset;
let ch = rest.chars().next().unwrap();
if line_start {
- let consumed = continuation(&mut result, rest, base, quote)?;
+ // Prose strips literal indentation after a soft line ending.
+ // A rendered leading space may instead come from an entity later
+ // on this line, so do not consume any parsed text for this gap.
+ let consumed = continuation_length(rest, quote);
line_start = false;
if consumed != 0 {
offset += consumed;
continue;
}
}
+ if matches!(ch, ' ' | '\t') {
+ let end = rest.len() - rest.trim_start_matches([' ', '\t']).len();
+ if rest[end..].starts_with(['\r', '\n']) {
+ // The Text tokenizer drops trailing literal whitespace;
+ // actual hard breaks have their own AST node and source map.
+ offset += end;
+ continue;
+ }
+ result.push(&rest[..end], base, base + end, MarkdownSourceKind::Literal)?;
+ offset += end;
+ continue;
+ }
if ch == '\\'
&& let Some(escaped) = rest.chars().nth(1)
&& escaped.is_ascii_punctuation()
@@ -192,13 +207,21 @@ fn reference(input: &str) -> Option<(usize, String)> {
let name = &input[1..end];
let decoded = if let Some(number) = name.strip_prefix("#x").or_else(|| name.strip_prefix("#X"))
{
- if number.is_empty() || !number.bytes().all(|byte| byte.is_ascii_hexdigit()) {
+ // Match markdown-rs' character-reference grammar (CommonMark), rather
+ // than decoding invalid long literals merely because u32 accepts them.
+ if number.is_empty()
+ || number.len() > 6
+ || !number.bytes().all(|byte| byte.is_ascii_hexdigit())
+ {
return None;
}
u32::from_str_radix(number, 16).ok()?;
::markdown::decode_numeric(number, 16)
} else if let Some(number) = name.strip_prefix('#') {
- if number.is_empty() || !number.bytes().all(|byte| byte.is_ascii_digit()) {
+ if number.is_empty()
+ || number.len() > 7
+ || !number.bytes().all(|byte| byte.is_ascii_digit())
+ {
return None;
}
number.parse::<u32>().ok()?;
@@ -236,7 +259,7 @@ pub(super) fn inline_code(
let ch = rest.chars().next().unwrap();
let base = start(node) + offset;
if line_start {
- let consumed = continuation(&mut result, rest, base, quote)?;
+ let consumed = code_continuation(&mut result, rest, base, quote)?;
line_start = false;
if consumed != 0 {
offset += consumed;
@@ -268,15 +291,7 @@ pub(super) fn inline_code(
result.finish()
}
-/// Container syntax and leading whitespace form one indentation token. Any
-/// whitespace retained by the parser belongs to that token, rather than to a
-/// guessed occurrence among several identical source spaces.
-fn continuation(
- result: &mut Projection<'_>,
- input: &str,
- base: usize,
- quote: u32,
-) -> Result<usize, String> {
+fn continuation_length(input: &str, quote: u32) -> usize {
let mut quotes = 0;
let mut consumed = 0;
for ch in input.chars() {
@@ -289,6 +304,18 @@ fn continuation(
break;
}
}
+ consumed
+}
+
+/// Inline code retains indentation left after its container prefix. Keep that
+/// whitespace as one lexical token rather than guessing among equal spaces.
+fn code_continuation(
+ result: &mut Projection<'_>,
+ input: &str,
+ base: usize,
+ quote: u32,
+) -> Result<usize, String> {
+ let consumed = continuation_length(input, quote);
if consumed != 0 {
let remaining = result.remaining();
let end = remaining.len() - remaining.trim_start_matches([' ', '\t']).len();
@@ -350,14 +377,13 @@ pub(super) fn code(
let rendered = result.remaining();
let rendered_end = rendered.find(['\r', '\n']).unwrap_or(rendered.len());
let rendered_line = &rendered[..rendered_end];
- let normalized = content.replace('\0', "�");
- let (prefix, leading) = if let Some(prefix) = normalized.strip_suffix(rendered_line) {
+ let (prefix, leading) = if let Some(prefix) = content.strip_suffix(rendered_line) {
(prefix, "")
} else {
// A tab consumed partly by indentation can leave rendered spaces.
// Keep that indentation as one lexical replacement, never the code.
let tail = rendered_line.trim_start_matches([' ', '\t']);
- let prefix = normalized
+ let prefix = content
.strip_suffix(tail)
.ok_or("Markdown code line is not an exact source suffix")?;
(prefix, &rendered_line[..rendered_line.len() - tail.len()])
@@ -375,29 +401,12 @@ pub(super) fn code(
)?;
}
let kept = &content[prefix.len()..];
- if !kept.contains('\0') {
- result.push(
- kept,
- base + prefix.len(),
- base + content.len(),
- MarkdownSourceKind::Literal,
- )?;
- } else {
- for (at, ch) in kept.char_indices() {
- let base = base + prefix.len() + at;
- if ch == '\0' && result.remaining().starts_with('�') {
- result.push("�", base, base + 1, MarkdownSourceKind::Replacement)?;
- } else {
- let text = &source[base..base + ch.len_utf8()];
- result.push(
- text,
- base,
- base + ch.len_utf8(),
- MarkdownSourceKind::Literal,
- )?;
- }
- }
- }
+ result.push(
+ kept,
+ base + prefix.len(),
+ base + content.len(),
+ MarkdownSourceKind::Literal,
+ )?;
if rendered_end < rendered.len() {
let ending = if rendered[rendered_end..].starts_with("\r\n") {
"\r\n"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org