You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maka.apache.org by wy...@apache.org on 2026/10/18 07:55:37 UTC
(maka) branch rust-preview updated: fix(files): stream paged text reads without a source size cap
This is an automated email from the ASF dual-hosted git repository.
M4n5ter pushed a commit to branch rust-preview
in repository https://gitbox.apache.org/repos/asf/maka.git
The following commit(s) were added to refs/heads/rust-preview by this push:
new f74e35127d fix(files): stream paged text reads without a source size cap
f74e35127d is described below
commit f74e35127da35dac771c49b2fbe0625741efeae5
Author: Wang <m4...@gmail.com>
AuthorDate: Sun Oct 18 15:55:30 2026 +0800
fix(files): stream paged text reads without a source size cap
Scan the authorized file incrementally while preserving complete UTF-8 and
NUL validation, line counts, cancellation, and content-bound continuations.
Remove ReadLimits and migrate all read callers to an explicit page budget.
Reserve the Native completed-result envelope so Unicode continuations remain
available to the model instead of being immediately archived.
Generated-by: Codex
---
plugins/agent/facts/src/read.rs | 2 +-
plugins/agent/facts/src/read/page.rs | 262 ++++++++++++++-------
plugins/agent/native/src/owner/tests/files.rs | 77 ++++++
plugins/workspace/workspace-files/engine/README.md | 4 +-
.../workspace-files/engine/README.zh-CN.md | 4 +-
.../workspace/workspace-files/engine/src/lib.rs | 119 +++++-----
.../workspace/workspace-files/engine/src/scoped.rs | 94 +++++---
.../workspace-files/engine/src/scoped/tests.rs | 51 ++++
.../engine/tests/directory_authorization.rs | 13 +-
.../workspace/workspace-files/engine/tests/glob.rs | 3 +-
.../workspace/workspace-files/engine/tests/grep.rs | 3 +-
.../engine/tests/grep_completeness.rs | 3 +-
.../workspace-files/engine/tests/grep_scope.rs | 6 +-
.../workspace-files/engine/tests/policy.rs | 11 +-
.../workspace/workspace-files/engine/tests/read.rs | 58 +++--
.../workspace-files/engine/tests/windows_read.rs | 10 +-
.../workspace-files/native/src/attachments.rs | 8 +-
.../workspace/workspace-files/native/src/lib.rs | 30 ++-
products/maka/cli/src/code.rs | 6 +-
19 files changed, 520 insertions(+), 244 deletions(-)
diff --git a/plugins/agent/facts/src/read.rs b/plugins/agent/facts/src/read.rs
index 6bec30e9fd..f53faf8767 100644
--- a/plugins/agent/facts/src/read.rs
+++ b/plugins/agent/facts/src/read.rs
@@ -22,7 +22,7 @@ mod page;
mod resource;
mod shell;
mod tool_result;
-pub use page::ReadPage;
+pub use page::{ReadPage, TextPageScan};
pub use resource::ResourceAddress;
pub use shell::ReadMetadata;
diff --git a/plugins/agent/facts/src/read/page.rs b/plugins/agent/facts/src/read/page.rs
index 541c212796..01c3e6a132 100644
--- a/plugins/agent/facts/src/read/page.rs
+++ b/plugins/agent/facts/src/read/page.rs
@@ -21,7 +21,7 @@ use super::{MAX_PAGE_CHARS, PREFIX, ReadError, ReadInput, ReadRequest, Start, UR
use base64::Engine;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
-use std::{cell::OnceCell, num::NonZeroUsize};
+use std::num::NonZeroUsize;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
@@ -41,9 +41,40 @@ pub struct ReadPage {
struct Position {
byte: usize,
utf16: usize,
+ line: usize,
+ line_start: usize,
+ before_newline: bool,
+}
+
+/// One text scan, retaining only candidates for the requested bounded page.
+/// The caller must feed the entire source before finishing, including its tail.
+pub struct TextPageScan<'a> {
+ request: &'a ReadRequest,
+ budget: usize,
+ digest: Sha256,
+ position: Position,
+ start: Option<Position>,
+ end: Option<Position>,
+ content: String,
+ candidates: Vec<Position>,
+ full: bool,
}
impl ReadRequest {
+ pub fn scan_with_budget(&self, budget: usize) -> TextPageScan<'_> {
+ TextPageScan {
+ request: self,
+ budget,
+ digest: Sha256::new(),
+ position: Position::default(),
+ start: None,
+ end: None,
+ content: String::new(),
+ candidates: Vec::new(),
+ full: false,
+ }
+ }
+
pub fn page(&self, text: &str) -> Result<ReadPage, ReadError> {
self.page_with_budget(text, MAX_PAGE_CHARS)
}
@@ -51,61 +82,103 @@ impl ReadRequest {
/// Budget is the serialized page's UTF-16 length, including continuation.
/// Callers adding execution metadata reserve its serialized size here.
pub fn page_with_budget(&self, text: &str, budget: usize) -> Result<ReadPage, ReadError> {
- let digest = OnceCell::new();
- let requested_position = match &self.start {
- Start::Line(_) => None,
- Start::Continuation {
- position,
- digest: expected,
- } => {
- let actual = digest.get_or_init(|| content_digest(text));
- if actual != expected {
- return Err(ReadError::ContentChanged);
- }
- Some(*position)
+ let mut scan = self.scan_with_budget(budget);
+ scan.push(text);
+ scan.finish()
+ }
+}
+
+impl TextPageScan<'_> {
+ fn select_start(&mut self) {
+ if self.start.is_some() {
+ return;
+ }
+ let matches = match self.request.start {
+ Start::Line(line) => {
+ self.position.line == line && self.position.byte == self.position.line_start
}
+ Start::Continuation { position, .. } => self.position.utf16 == position,
};
- let mut lines = vec![Position::default()];
- let mut position = Position::default();
- let mut resumed = None;
- for (byte, character) in text.char_indices() {
- if requested_position == Some(position.utf16) {
- resumed = Some(position);
+ if matches {
+ self.start = Some(self.position);
+ self.candidates.push(self.position);
+ }
+ }
+
+ pub fn push(&mut self, text: &str) {
+ self.digest.update(text.as_bytes());
+ for character in text.chars() {
+ self.select_start();
+ self.position.before_newline = character == '\n';
+ if let Some(last) = self.candidates.last_mut()
+ && last.byte == self.position.byte
+ {
+ last.before_newline = self.position.before_newline;
}
- position = Position {
- byte: byte + character.len_utf8(),
- utf16: position.utf16 + character.len_utf16(),
- };
+ if let Some(start) = self.start {
+ if self.end.is_none()
+ && character == '\n'
+ && self.request.limit.is_some_and(|limit| {
+ self.position.line + 1 == start.line.saturating_add(limit.get())
+ })
+ {
+ self.end = Some(self.position);
+ }
+ if self.end.is_none() && !self.full {
+ if self.position.utf16 - start.utf16 + character.len_utf16() <= self.budget {
+ self.content.push(character);
+ } else {
+ self.full = true;
+ }
+ }
+ }
+ self.position.byte += character.len_utf8();
+ self.position.utf16 += character.len_utf16();
if character == '\n' {
- lines.push(position);
+ self.position.line += 1;
+ self.position.line_start = self.position.byte;
+ }
+ self.position.before_newline = false;
+ if self.start.is_some() && self.end.is_none() && !self.full {
+ self.candidates.push(self.position);
}
}
- if requested_position == Some(position.utf16) {
- resumed = Some(position);
- }
- let line_at = |byte| lines.partition_point(|line| line.byte <= byte) - 1;
- let (offset, start) = match self.start {
- Start::Line(offset) => (offset, lines.get(offset).copied().unwrap_or(position)),
- Start::Continuation { .. } => {
- let start = resumed.ok_or(ReadError::InvalidPosition)?;
- (line_at(start.byte), start)
+ }
+
+ pub fn finish(mut self) -> Result<ReadPage, ReadError> {
+ self.select_start();
+ let digest = hex::encode(self.digest.finalize());
+ let digest = &digest[..32];
+ if let Start::Continuation {
+ digest: expected, ..
+ } = &self.request.start
+ {
+ if digest != expected {
+ return Err(ReadError::ContentChanged);
+ }
+ if self.start.is_none() {
+ return Err(ReadError::InvalidPosition);
}
+ }
+ let start = self.start.unwrap_or(self.position);
+ if self.candidates.is_empty() {
+ self.candidates.push(start);
+ }
+ let offset = match self.request.start {
+ Start::Line(offset) => offset,
+ Start::Continuation { .. } => start.line,
};
+ let total_lines = self.position.line + 1;
let last_line = offset
- .saturating_add(self.limit.map_or(lines.len(), NonZeroUsize::get))
- .min(lines.len());
- let end = lines.get(last_line).map_or(position, |line| Position {
- byte: line.byte - 1,
- utf16: line.utf16 - 1,
- });
+ .saturating_add(self.request.limit.map_or(total_lines, NonZeroUsize::get))
+ .min(total_lines);
+ let end = self.end.unwrap_or(self.position);
let make_page = |stop: Position| {
let complete = stop.byte >= end.byte;
- let next_line = offset.max(line_at(stop.byte));
- let boundary = lines
- .get(next_line + 1)
- .is_some_and(|line| stop.byte == line.byte - 1);
+ let next_line = offset.max(stop.line);
+ let boundary = stop.before_newline && stop.line == next_line;
let partial = !complete && !boundary;
- let returned_lines = if offset >= lines.len() {
+ let returned_lines = if offset >= total_lines {
0
} else if complete {
last_line - offset
@@ -114,20 +187,19 @@ impl ReadRequest {
};
ReadPage {
metadata: None,
- content: text[start.byte..stop.byte.max(start.byte)].into(),
+ content: self.content[..stop.byte.saturating_sub(start.byte)].into(),
offset,
returned_lines,
- total_lines: lines.len(),
- partial_line: partial || (offset < lines.len() && start.byte > lines[offset].byte),
+ total_lines,
+ partial_line: partial || (offset < total_lines && start.byte > start.line_start),
next: (!complete).then(|| ReadInput {
path: format!(
- "{PREFIX}{}?at={}&sha={}",
- URL_SAFE_NO_PAD.encode(&self.path),
- stop.utf16 + usize::from(!partial),
- digest.get_or_init(|| content_digest(text))
+ "{PREFIX}{}?at={}&sha={digest}",
+ URL_SAFE_NO_PAD.encode(&self.request.path),
+ stop.utf16 + usize::from(!partial)
),
offset: None,
- limit: self.limit.map(|limit| {
+ limit: self.request.limit.map(|limit| {
NonZeroUsize::new(limit.get() - returned_lines)
.expect("incomplete line range")
}),
@@ -139,32 +211,18 @@ impl ReadRequest {
.expect("ReadPage contains only JSON fields")
.encode_utf16()
.count()
- <= budget
+ <= self.budget
};
- if end.utf16.saturating_sub(start.utf16) <= budget {
+ if end.utf16.saturating_sub(start.utf16) <= self.budget {
let page = make_page(end);
if fits(&page) {
return Ok(page);
}
}
- // Only retain candidate boundaries for this bounded page, not every
- // character in a potentially large source. Rust slices remain UTF-8 safe.
- let mut candidates = vec![start];
- let mut utf16 = start.utf16;
- for (relative, character) in text[start.byte..end.byte].char_indices() {
- utf16 += character.len_utf16();
- if utf16 - start.utf16 > budget {
- break;
- }
- candidates.push(Position {
- byte: start.byte + relative + character.len_utf8(),
- utf16,
- });
- }
- let (mut low, mut high) = (0, candidates.len() - 1);
+ let (mut low, mut high) = (0, self.candidates.len() - 1);
while low < high {
let middle = low + (high - low).div_ceil(2);
- if fits(&make_page(candidates[middle])) {
+ if fits(&make_page(self.candidates[middle])) {
low = middle;
} else {
high = middle - 1;
@@ -173,18 +231,16 @@ impl ReadRequest {
if low == 0 {
return Err(ReadError::AddressTooLong);
}
- let mut stop = candidates[low];
- if let Some(relative) = text[start.byte..stop.byte]
+ let mut stop = self.candidates[low];
+ if let Some(relative) = self.content[..stop.byte - start.byte]
.rfind('\n')
- .filter(|relative| *relative > 0 && text.as_bytes().get(stop.byte) != Some(&b'\n'))
+ .filter(|relative| *relative > 0 && !stop.before_newline)
{
- stop = Position {
- byte: start.byte + relative,
- utf16: start.utf16
- + text[start.byte..start.byte + relative]
- .encode_utf16()
- .count(),
- };
+ stop = *self
+ .candidates
+ .iter()
+ .find(|candidate| candidate.byte == start.byte + relative)
+ .expect("retained scalar boundary");
}
let page = make_page(stop);
if !fits(&page) {
@@ -194,6 +250,7 @@ impl ReadRequest {
}
}
+#[cfg(test)]
fn content_digest(text: &str) -> String {
hex::encode(Sha256::digest(text.as_bytes()))[..32].into()
}
@@ -240,6 +297,18 @@ mod tests {
.unwrap()
.page_with_budget(&text, 512)
.unwrap();
+ for chunk_bytes in [1, 7, 8192] {
+ let request = input.resolve().unwrap();
+ let mut scan = request.scan_with_budget(512);
+ let mut start = 0;
+ while start < text.len() {
+ let end =
+ text.ceil_char_boundary((start + chunk_bytes).min(text.len()));
+ scan.push(&text[start..end]);
+ start = end;
+ }
+ assert_eq!(scan.finish().unwrap(), page);
+ }
assert!(serde_json::to_string(&page).unwrap().encode_utf16().count() <= 512);
actual.push_str(&page.content);
consumed += page.content.encode_utf16().count();
@@ -286,4 +355,37 @@ mod tests {
Err(ReadError::AddressTooLong)
);
}
+
+ #[test]
+ fn every_scalar_cut_preserves_literal_bom_nul_and_line_metadata() {
+ let text = "\u{feff}零\0😀\r\n尾\n";
+ let input: ReadInput = serde_json::from_value(json!({"path":"file"})).unwrap();
+ let request = input.resolve().unwrap();
+ for cut in text
+ .char_indices()
+ .map(|(byte, _)| byte)
+ .chain([text.len()])
+ {
+ let mut scan = request.scan_with_budget(512);
+ scan.push(&text[..cut]);
+ scan.push(&text[cut..]);
+ assert_eq!(
+ serde_json::to_value(scan.finish().unwrap()).unwrap(),
+ json!({
+ "content":"\u{feff}零\0😀\r\n尾\n","offset":0,"returnedLines":3,"totalLines":3,"next":null
+ })
+ );
+ }
+ let request = ReadRequest {
+ path: "file".into(),
+ start: Start::Continuation {
+ position: 999,
+ digest: "0".repeat(32),
+ },
+ limit: None,
+ };
+ let mut scan = request.scan_with_budget(512);
+ scan.push(text);
+ assert_eq!(scan.finish(), Err(ReadError::ContentChanged));
+ }
}
diff --git a/plugins/agent/native/src/owner/tests/files.rs b/plugins/agent/native/src/owner/tests/files.rs
index c3798f3280..a71ff0e9ce 100644
--- a/plugins/agent/native/src/owner/tests/files.rs
+++ b/plugins/agent/native/src/owner/tests/files.rs
@@ -160,6 +160,83 @@ async fn close(f: &Fixture) {
f.config.shutdown().await.unwrap();
}
+#[tokio::test]
+async fn native_read_text_pages_source_larger_than_one_mib() {
+ tokio::time::timeout(Duration::from_secs(60), async {
+ let (listener, f, session) = fixture("readText").await;
+ let source: String = (0..65_537)
+ .map(|line| format!("row-{line:08}-data\n"))
+ .collect();
+ let source_bytes = source.len();
+ std::fs::write(f.workspace.join("large-numbered.txt"), &source).unwrap();
+ let mut ranges = Vec::new();
+ for offset in [0, 32_768, 65_536, 65_538] {
+ call(&listener, &f, &session,
+ json!({"path":"large-numbered.txt","offset":offset,"limit":1})).await;
+ ranges.push(finish(&listener, &f).await);
+ }
+ let line = "中文😀\\\"\t".repeat(2_000);
+ let original = format!("prefix\n{line}\n{source}");
+ std::fs::write(f.workspace.join("large-unicode.txt"), &original).unwrap();
+ let mut input = json!({"path":"large-unicode.txt","offset":1,"limit":1});
+ let mut pages = Vec::new();
+ loop {
+ call(&listener, &f, &session, input).await;
+ let page = finish(&listener, &f).await;
+ let next = page["value"]["next"].clone();
+ pages.push(page);
+ if next.is_null() { break; }
+ input = next;
+ }
+ let first_next = pages[0]["value"]["next"].clone();
+ let changed = if !first_next.is_null() {
+ std::fs::write(f.workspace.join("large-unicode.txt"),
+ original.replace("row-00065536-data", "row-99965536-data")).unwrap();
+ call(&listener, &f, &session, first_next).await;
+ Some(finish(&listener, &f).await)
+ } else { None };
+ call(&listener, &f, &session,
+ json!({"path":"large-unicode.txt","offset":1,"limit":1})).await;
+ let fresh = finish(&listener, &f).await;
+ // All public result assertions follow real Host/Agent owner cleanup.
+ close(&f).await;
+ assert_eq!(source_bytes, 1_179_666);
+ for (result, (offset, content, returned)) in ranges.iter().zip([
+ (0, "row-00000000-data", 1),
+ (32_768, "row-00032768-data", 1),
+ (65_536, "row-00065536-data", 1),
+ (65_538, "", 0),
+ ]) {
+ assert_eq!(result, &json!({"kind":"completed","value":{
+ "content":content,"offset":offset,"returnedLines":returned,"totalLines":65_538,"next":null
+ }}));
+ }
+ assert!(pages.len() > 1, "long Unicode line must exercise actual continuations: {pages:?}");
+ let mut reconstructed = String::new();
+ for (index, page) in pages.iter().enumerate() {
+ assert_eq!(page["kind"], "completed", "{page}");
+ assert!(serde_json::to_string(page).unwrap().encode_utf16().count()
+ <= maka_agent_facts::read::MAX_PAGE_CHARS);
+ let value = &page["value"];
+ assert!(serde_json::to_string(value).unwrap().encode_utf16().count()
+ <= maka_agent_facts::read::MAX_PAGE_CHARS);
+ assert_eq!(value["offset"], 1);
+ assert_eq!(value["totalLines"], 65_540);
+ assert_eq!(value["partialLine"], true);
+ assert_eq!(value["returnedLines"], usize::from(index + 1 == pages.len()));
+ reconstructed.push_str(value["content"].as_str().unwrap());
+ }
+ assert!(reconstructed == line, "public continuations changed the original Unicode bytes");
+ assert_eq!(changed, Some(json!({"kind":"failed","message":
+ "The content changed since the previous page. Read the original path again"})));
+ assert_eq!(fresh["kind"], "completed", "{fresh}");
+ assert_eq!(fresh["value"]["content"], pages[0]["value"]["content"]);
+ assert_ne!(fresh["value"]["next"], pages[0]["value"]["next"]);
+ })
+ .await
+ .unwrap();
+}
+
#[tokio::test]
async fn native_files_reuse_actual_read_search_write_edit_and_patch_executors() {
tokio::time::timeout(Duration::from_secs(90), async {
diff --git a/plugins/workspace/workspace-files/engine/README.md b/plugins/workspace/workspace-files/engine/README.md
index d56681b2ab..574caeb851 100644
--- a/plugins/workspace/workspace-files/engine/README.md
+++ b/plugins/workspace/workspace-files/engine/README.md
@@ -21,7 +21,9 @@
[简体中文](README.zh-CN.md)
-Implements file reads, search, edits, writes, patches and worktree operations. Callers provide the authorized filesystem scope. `ReadExecutor` handles bounded reads; `MutationExecutor` and `WriteCoordinator` prepare and serialize writes. Patch parsing is delegated to the adjacent `patch` crate.
+Implements file reads, search, edits, writes, patches and worktree operations. Callers provide the authorized filesystem scope. `ReadExecutor` handles paged reads; `MutationExecutor` and `WriteCoordinator` prepare and serialize writes. Patch parsing is delegated to the adjacent `patch` crate.
+
+Text reads scan the original authorized file descriptor once per page, retaining only a bounded page candidate. The serialized response budget does not limit total text size. Every page validates the complete UTF-8 source (rejecting NUL), counts all lines and binds continuations to its content digest; reading later pages repeats that full scan. Cancellation is checked between chunks and the caller awaits the worker. This does not provide an atomic snapshot of concurrently edited files. I [...]
`workspace` prepares intrinsic identity through the original captured directory. For a usable Git repository, it appends `.maka-workspace.json` to the common `info/exclude` only when absent, preserving existing bytes. This fixed preparation covers selected subdirectories and linked worktrees; it requires the real private Root/control boundaries and an inspected binding for managed worktrees. An unusable nearest `.git` entry does not redirect preparation to an ancestor repository; failure [...]
diff --git a/plugins/workspace/workspace-files/engine/README.zh-CN.md b/plugins/workspace/workspace-files/engine/README.zh-CN.md
index 82df0e69b0..d54dba1988 100644
--- a/plugins/workspace/workspace-files/engine/README.zh-CN.md
+++ b/plugins/workspace/workspace-files/engine/README.zh-CN.md
@@ -21,7 +21,9 @@
[English](README.md)
-实现文件读取、搜索、编辑、写入、补丁和 worktree 操作。调用方提供已授权的文件系统作用域。`ReadExecutor` 处理有界读取,`MutationExecutor` 与 `WriteCoordinator` 准备并串行化写入。补丁解析委托相邻的 `patch` crate。
+实现文件读取、搜索、编辑、写入、补丁和 worktree 操作。调用方提供已授权的文件系统作用域。`ReadExecutor` 处理分页读取,`MutationExecutor` 与 `WriteCoordinator` 准备并串行化写入。补丁解析委托相邻的 `patch` crate。
+
+文本读取每页通过原始已授权文件描述符扫描一次,仅保留有界的页面候选内容。序列化响应预算不限制文本总大小。每页都会验证完整源文件的 UTF-8(拒绝 NUL)、统计所有行,并将续页地址绑定到内容摘要;后续页仍须重新扫描完整文件。读取在块之间检查取消,调用方等待工作线程结束。这不保证并发修改文件的原子快照。图像读取保留独立的字节与格式限制。
`workspace` 通过原始已捕获目录准备内在身份。存在可用 Git 仓库时,仅在缺少规则时向共用的 `info/exclude` 追加 `.maka-workspace.json`,保留已有字节。该固定初始化操作覆盖选中的子目录和 linked worktree,并要求真实私有 Root/control 边界及受检验的托管 worktree binding。最近的 `.git` 入口不可用时不会转而修改祖先仓库;向有效排除文件写入失败仍会报错。已跟踪文件和用户忽略规则保留 Git 原有的优先级。
diff --git a/plugins/workspace/workspace-files/engine/src/lib.rs b/plugins/workspace/workspace-files/engine/src/lib.rs
index bcb3ac5b39..42335da297 100644
--- a/plugins/workspace/workspace-files/engine/src/lib.rs
+++ b/plugins/workspace/workspace-files/engine/src/lib.rs
@@ -71,19 +71,6 @@ pub enum ReadScope {
Unrestricted,
}
-#[derive(Clone, Copy)]
-pub struct ReadLimits {
- pub max_source_bytes: usize,
-}
-
-impl Default for ReadLimits {
- fn default() -> Self {
- Self {
- max_source_bytes: 1024 * 1024,
- }
- }
-}
-
/// Bytes remain internal until the Host commits the image snapshot and result.
#[derive(Debug, PartialEq, Eq)]
pub enum ReadOutput {
@@ -94,7 +81,6 @@ pub enum ReadOutput {
#[derive(Clone)]
pub struct ReadExecutor {
authority: Arc<scoped::Authority>,
- limits: ReadLimits,
}
impl ReadExecutor {
@@ -103,43 +89,30 @@ impl ReadExecutor {
pub fn from_directory(
path: PathBuf,
directory: cap_std::fs::Dir,
- limits: ReadLimits,
policy: Option<Arc<maka_sandbox::filesystem::Compiled>>,
) -> Result<Self, ToolError> {
- if limits.max_source_bytes == 0 || limits.max_source_bytes.checked_add(1).is_none() {
- return Err(failed("Read byte limits must be positive and bounded"));
- }
Ok(Self {
authority: Arc::new(scoped::Authority::from_directory(path, directory, policy)?),
- limits,
})
}
- pub fn new(
- cwd: impl AsRef<Path>,
- scope: ReadScope,
- limits: ReadLimits,
- ) -> Result<Self, ToolError> {
- if limits.max_source_bytes == 0 || limits.max_source_bytes.checked_add(1).is_none() {
- return Err(failed("Read byte limits must be positive and bounded"));
- }
+ pub fn new(cwd: impl AsRef<Path>, scope: ReadScope) -> Result<Self, ToolError> {
Ok(Self {
authority: Arc::new(scoped::Authority::new(cwd.as_ref(), scope)?),
- limits,
})
}
pub async fn read(
&self,
input: ReadRequest,
+ page_budget: usize,
cancellation: CancellationToken,
) -> Result<ReadOutput, ToolError> {
let authority = self.authority.clone();
- let limits = self.limits;
if input.path().contains("://") {
return Err(failed("Filesystem Read requires a file path"));
}
// Always join blocking I/O: cancellation must not detach unsettled work.
- tokio::task::spawn_blocking(move || authority.read(input, limits, cancellation))
+ tokio::task::spawn_blocking(move || authority.read(input, page_budget, cancellation))
.await
.map_err(|e| failed(format!("Read worker failed: {e}")))?
}
@@ -165,7 +138,14 @@ impl ToolExecutor for ReadExecutor {
let input: ReadInput =
serde_json::from_value(input).map_err(|e| failed(e.to_string()))?;
let request = input.resolve().map_err(|e| failed(e.to_string()))?;
- match executor.read(request, cancellation).await? {
+ match executor
+ .read(
+ request,
+ maka_agent_facts::read::MAX_PAGE_CHARS,
+ cancellation,
+ )
+ .await?
+ {
ReadOutput::Text(page) => Ok(serde_json::to_value(page).expect("ReadPage is JSON")),
ReadOutput::Image { .. } => Err(failed(
"Read image requires a durable journal snapshot; use the Host typed Read path",
@@ -208,10 +188,6 @@ mod image_tests {
ReadScope::Restricted {
roots: vec![root.to_owned()],
},
- // Images have an independent cap, including when text limits are tiny.
- ReadLimits {
- max_source_bytes: 1,
- },
)
.unwrap()
}
@@ -227,7 +203,7 @@ mod image_tests {
("picture.webp", b"RIFF\x16\0\0\0WEBPVP8X\x0a\0\0\0\0\0\0\0\0\0\0\0\0\0".to_vec(), "image/webp"),
] {
fs::write(temp.path().join(path), &bytes).unwrap();
- assert_eq!(read.read(request(path), CancellationToken::new()).await.unwrap(),
+ assert_eq!(read.read(request(path), maka_agent_facts::read::MAX_PAGE_CHARS, CancellationToken::new()).await.unwrap(),
ReadOutput::Image { bytes, mime_type: mime_type.into() });
}
let error = read
@@ -250,9 +226,13 @@ mod image_tests {
at_cap.resize(5 * 1024 * 1024, 0);
fs::write(&path, &at_cap).unwrap();
assert!(
- read.read(request(&path), CancellationToken::new())
- .await
- .is_ok()
+ read.read(
+ request(&path),
+ maka_agent_facts::read::MAX_PAGE_CHARS,
+ CancellationToken::new()
+ )
+ .await
+ .is_ok()
);
at_cap.push(0);
let mut corrupt_png = png(1, 1);
@@ -271,15 +251,27 @@ mod image_tests {
] {
fs::write(&path, bytes).unwrap();
assert!(
- read.read(request(&path), CancellationToken::new())
- .await
- .is_err()
+ read.read(
+ request(&path),
+ maka_agent_facts::read::MAX_PAGE_CHARS,
+ CancellationToken::new()
+ )
+ .await
+ .is_err()
);
}
fs::write(&path, png(1, 1)).unwrap();
let cancellation = CancellationToken::new();
cancellation.cancel();
- assert!(read.read(request(path), cancellation).await.is_err());
+ assert!(
+ read.read(
+ request(path),
+ maka_agent_facts::read::MAX_PAGE_CHARS,
+ cancellation
+ )
+ .await
+ .is_err()
+ );
}
#[tokio::test]
@@ -300,13 +292,16 @@ mod image_tests {
ReadScope::Restricted {
roots: vec![alias.clone()],
},
- ReadLimits::default(),
)
.unwrap();
assert_eq!(
- read.read(request(alias.join("alias.png")), CancellationToken::new())
- .await
- .unwrap(),
+ read.read(
+ request(alias.join("alias.png")),
+ maka_agent_facts::read::MAX_PAGE_CHARS,
+ CancellationToken::new()
+ )
+ .await
+ .unwrap(),
ReadOutput::Image {
bytes: bytes.clone(),
mime_type: "image/png".into()
@@ -318,17 +313,25 @@ mod image_tests {
root.clone(),
] {
assert!(
- read.read(request(path), CancellationToken::new())
- .await
- .is_err()
+ read.read(
+ request(path),
+ maka_agent_facts::read::MAX_PAGE_CHARS,
+ CancellationToken::new()
+ )
+ .await
+ .is_err()
);
}
for name in ["data", "data.svg", "data.avif", "data.pdf"] {
fs::write(root.join(name), &bytes).unwrap();
assert!(
- read.read(request(name), CancellationToken::new())
- .await
- .is_err()
+ read.read(
+ request(name),
+ maka_agent_facts::read::MAX_PAGE_CHARS,
+ CancellationToken::new()
+ )
+ .await
+ .is_err()
);
}
// Replacing the ambient root cannot redirect an already captured capability.
@@ -336,9 +339,13 @@ mod image_tests {
fs::create_dir(&root).unwrap();
fs::write(root.join("alias.png"), "attacker").unwrap();
assert_eq!(
- read.read(request(alias.join("alias.png")), CancellationToken::new())
- .await
- .unwrap(),
+ read.read(
+ request(alias.join("alias.png")),
+ maka_agent_facts::read::MAX_PAGE_CHARS,
+ CancellationToken::new()
+ )
+ .await
+ .unwrap(),
ReadOutput::Image {
bytes,
mime_type: "image/png".into()
diff --git a/plugins/workspace/workspace-files/engine/src/scoped.rs b/plugins/workspace/workspace-files/engine/src/scoped.rs
index 788d619ae8..2a5bbbdacd 100644
--- a/plugins/workspace/workspace-files/engine/src/scoped.rs
+++ b/plugins/workspace/workspace-files/engine/src/scoped.rs
@@ -17,7 +17,7 @@
* under the License.
*/
-use crate::{ReadLimits, ReadOutput, ReadScope, failed};
+use crate::{ReadOutput, ReadScope, failed};
#[cfg(unix)]
use cap_std::fs::OpenOptionsExt;
use cap_std::{
@@ -327,7 +327,7 @@ impl Authority {
pub(crate) fn read(
&self,
input: ReadRequest,
- limits: ReadLimits,
+ page_budget: usize,
cancellation: CancellationToken,
) -> Result<ReadOutput, ToolError> {
check_cancelled(&cancellation)?;
@@ -342,11 +342,6 @@ impl Authority {
.iter()
.any(|known| ext.eq_ignore_ascii_case(known))
});
- let max_source_bytes = if image {
- MAX_IMAGE_BYTES
- } else {
- limits.max_source_bytes
- };
// Keep parent components intact for capability resolution, so symlink/..
// semantics cannot be changed by a lexical normalization.
let absolute = path::resolve(&self.cwd, path)?;
@@ -355,25 +350,25 @@ impl Authority {
if !metadata.is_file() {
return Err(failed("Read supports only regular files"));
}
- if metadata.len() > max_source_bytes as u64 {
- return Err(failed("Read source exceeds byte limit"));
- }
- let mut bytes = Vec::new();
- let mut chunk = [0_u8; 8192];
- loop {
- check_cancelled(&cancellation)?;
- let available = (max_source_bytes + 1 - bytes.len()).min(chunk.len());
- let count = file.read(&mut chunk[..available]).map_err(io_error)?;
- if count == 0 {
- break;
- }
- bytes.extend_from_slice(&chunk[..count]);
- if bytes.len() > max_source_bytes {
+ if image {
+ if metadata.len() > MAX_IMAGE_BYTES as u64 {
return Err(failed("Read source exceeds byte limit"));
}
- }
- check_cancelled(&cancellation)?;
- if image {
+ let mut bytes = Vec::new();
+ let mut chunk = [0_u8; 8192];
+ loop {
+ check_cancelled(&cancellation)?;
+ let available = (MAX_IMAGE_BYTES + 1 - bytes.len()).min(chunk.len());
+ let count = file.read(&mut chunk[..available]).map_err(io_error)?;
+ if count == 0 {
+ break;
+ }
+ bytes.extend_from_slice(&chunk[..count]);
+ if bytes.len() > MAX_IMAGE_BYTES {
+ return Err(failed("Read source exceeds byte limit"));
+ }
+ }
+ check_cancelled(&cancellation)?;
let mime_type = crate::image::validate(&bytes)?;
check_cancelled(&cancellation)?;
return Ok(ReadOutput::Image {
@@ -381,16 +376,54 @@ impl Authority {
mime_type: mime_type.into(),
});
}
- let content = String::from_utf8(bytes).map_err(|_| failed("Read requires UTF-8 text"))?;
- if content.contains('\0') {
- return Err(failed("Read does not support binary files"));
- }
- let page = input.page(&content).map_err(|e| failed(e.to_string()))?;
+ let page = text_page(&mut file, &input, page_budget, &cancellation)?;
check_cancelled(&cancellation)?;
Ok(ReadOutput::Text(page))
}
}
+fn text_page(
+ file: &mut impl Read,
+ input: &ReadRequest,
+ page_budget: usize,
+ cancellation: &CancellationToken,
+) -> Result<maka_agent_facts::read::ReadPage, ToolError> {
+ let mut scan = input.scan_with_budget(page_budget);
+ let mut buffer = [0_u8; 8192 + 3];
+ let mut carry = 0;
+ let mut binary = false;
+ loop {
+ check_cancelled(cancellation)?;
+ let count = file
+ .read(&mut buffer[carry..carry + 8192])
+ .map_err(io_error)?;
+ if count == 0 {
+ break;
+ }
+ let length = carry + count;
+ let text = match std::str::from_utf8(&buffer[..length]) {
+ Ok(text) => text,
+ Err(error) if error.error_len().is_none() => {
+ std::str::from_utf8(&buffer[..error.valid_up_to()]).expect("validated UTF-8 prefix")
+ }
+ Err(_) => return Err(failed("Read requires UTF-8 text")),
+ };
+ let valid = text.len();
+ binary |= text.contains('\0');
+ scan.push(text);
+ carry = length - valid;
+ buffer.copy_within(valid..length, 0);
+ }
+ check_cancelled(cancellation)?;
+ if carry != 0 {
+ return Err(failed("Read requires UTF-8 text"));
+ }
+ if binary {
+ return Err(failed("Read does not support binary files"));
+ }
+ scan.finish().map_err(|error| failed(error.to_string()))
+}
+
fn check_cancelled(token: &CancellationToken) -> Result<(), ToolError> {
if token.is_cancelled() {
Err(failed("Read cancelled"))
@@ -405,3 +438,6 @@ fn io_error(error: std::io::Error) -> ToolError {
message: format!("Read filesystem error: {error}"),
}
}
+
+#[cfg(test)]
+mod tests;
diff --git a/plugins/workspace/workspace-files/engine/src/scoped/tests.rs b/plugins/workspace/workspace-files/engine/src/scoped/tests.rs
new file mode 100644
index 0000000000..a6edde6c46
--- /dev/null
+++ b/plugins/workspace/workspace-files/engine/src/scoped/tests.rs
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use super::*;
+
+#[test]
+fn text_scan_observes_cancellation_after_a_real_read_before_publishing_a_page() {
+ struct Source {
+ cancellation: CancellationToken,
+ reads: usize,
+ }
+ impl Read for Source {
+ fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
+ self.reads += 1;
+ buffer[..6].copy_from_slice(b"first\n");
+ self.cancellation.cancel();
+ Ok(6)
+ }
+ }
+ let cancellation = CancellationToken::new();
+ let mut source = Source {
+ cancellation: cancellation.clone(),
+ reads: 0,
+ };
+ let input: maka_agent_facts::read::ReadInput =
+ serde_json::from_value(serde_json::json!({"path":"file","limit":1})).unwrap();
+ let result = text_page(
+ &mut source,
+ &input.resolve().unwrap(),
+ maka_agent_facts::read::MAX_PAGE_CHARS,
+ &cancellation,
+ );
+ assert!(matches!(result, Err(ToolError::Failed(message)) if message == "Read cancelled"));
+ assert_eq!(source.reads, 1);
+}
diff --git a/plugins/workspace/workspace-files/engine/tests/directory_authorization.rs b/plugins/workspace/workspace-files/engine/tests/directory_authorization.rs
index a8f0e1daf2..93c338e875 100644
--- a/plugins/workspace/workspace-files/engine/tests/directory_authorization.rs
+++ b/plugins/workspace/workspace-files/engine/tests/directory_authorization.rs
@@ -20,7 +20,7 @@
#[cfg(unix)]
use maka_agent_facts::read::ReadInput;
use maka_agent_facts::tools::ToolExecutor;
-use maka_fs_tools::{MutationExecutor, ReadExecutor, ReadLimits, WriteCoordinator, directory};
+use maka_fs_tools::{MutationExecutor, ReadExecutor, WriteCoordinator, directory};
use serde_json::json;
use std::sync::Arc;
@@ -33,13 +33,8 @@ async fn directory_consent_does_not_mark_workspaces_or_follow_replacements() {
let (location, identity) = directory::capture(&root).unwrap();
assert!(!root.join(".maka-workspace.json").exists());
let handle = directory::open(&location, &identity).unwrap();
- let read = ReadExecutor::from_directory(
- location.clone(),
- handle.try_clone().unwrap(),
- ReadLimits::default(),
- None,
- )
- .unwrap();
+ let read =
+ ReadExecutor::from_directory(location.clone(), handle.try_clone().unwrap(), None).unwrap();
let write = MutationExecutor::from_directory(
location.clone(),
handle,
@@ -76,6 +71,7 @@ async fn directory_consent_does_not_mark_workspaces_or_follow_replacements() {
}
.resolve()
.unwrap(),
+ maka_agent_facts::read::MAX_PAGE_CHARS,
Default::default(),
)
.await
@@ -145,7 +141,6 @@ async fn policy_directory_allows_external_access_without_following_workspace_rep
let read = ReadExecutor::from_directory(
location.clone(),
handle.try_clone().unwrap(),
- ReadLimits::default(),
Some(policy.clone()),
)
.unwrap();
diff --git a/plugins/workspace/workspace-files/engine/tests/glob.rs b/plugins/workspace/workspace-files/engine/tests/glob.rs
index c9f96dfa74..de3ef3bc03 100644
--- a/plugins/workspace/workspace-files/engine/tests/glob.rs
+++ b/plugins/workspace/workspace-files/engine/tests/glob.rs
@@ -18,7 +18,7 @@
*/
use maka_agent_facts::tools::ToolExecutor;
-use maka_fs_tools::{ReadExecutor, ReadLimits, ReadScope};
+use maka_fs_tools::{ReadExecutor, ReadScope};
use serde_json::json;
#[cfg(unix)]
use std::os::unix::fs::symlink;
@@ -31,7 +31,6 @@ fn reader(root: &Path) -> ReadExecutor {
ReadScope::Restricted {
roots: vec![root.to_owned()],
},
- ReadLimits::default(),
)
.unwrap()
}
diff --git a/plugins/workspace/workspace-files/engine/tests/grep.rs b/plugins/workspace/workspace-files/engine/tests/grep.rs
index 2959a94848..0a21b995da 100644
--- a/plugins/workspace/workspace-files/engine/tests/grep.rs
+++ b/plugins/workspace/workspace-files/engine/tests/grep.rs
@@ -18,7 +18,7 @@
*/
use maka_agent_facts::tools::ToolExecutor;
-use maka_fs_tools::{ReadExecutor, ReadLimits, ReadScope};
+use maka_fs_tools::{ReadExecutor, ReadScope};
use serde_json::{Value, json};
#[cfg(unix)]
use std::os::unix::{ffi::OsStrExt, fs::symlink};
@@ -31,7 +31,6 @@ fn reader(root: &Path) -> ReadExecutor {
ReadScope::Restricted {
roots: vec![root.to_owned()],
},
- ReadLimits::default(),
)
.unwrap()
}
diff --git a/plugins/workspace/workspace-files/engine/tests/grep_completeness.rs b/plugins/workspace/workspace-files/engine/tests/grep_completeness.rs
index 0c73273025..8732aab546 100644
--- a/plugins/workspace/workspace-files/engine/tests/grep_completeness.rs
+++ b/plugins/workspace/workspace-files/engine/tests/grep_completeness.rs
@@ -18,7 +18,7 @@
*/
use maka_agent_facts::tools::ToolExecutor;
-use maka_fs_tools::{ReadExecutor, ReadLimits, ReadScope};
+use maka_fs_tools::{ReadExecutor, ReadScope};
use serde_json::json;
use std::fs;
use tokio_util::sync::CancellationToken;
@@ -32,7 +32,6 @@ async fn exact_caps_and_unscanned_paths_have_explicit_completeness() {
ReadScope::Restricted {
roots: vec![root.clone()],
},
- ReadLimits::default(),
)
.unwrap();
for (lines, complete) in [
diff --git a/plugins/workspace/workspace-files/engine/tests/grep_scope.rs b/plugins/workspace/workspace-files/engine/tests/grep_scope.rs
index 91a8ca4b29..e5c406a908 100644
--- a/plugins/workspace/workspace-files/engine/tests/grep_scope.rs
+++ b/plugins/workspace/workspace-files/engine/tests/grep_scope.rs
@@ -18,7 +18,7 @@
*/
use maka_agent_facts::tools::ToolExecutor;
-use maka_fs_tools::{ReadExecutor, ReadLimits, ReadScope};
+use maka_fs_tools::{ReadExecutor, ReadScope};
use serde_json::json;
use std::{fs, process::Command};
use tokio_util::sync::CancellationToken;
@@ -57,9 +57,7 @@ async fn slash_glob_uses_session_cwd_independently_of_the_granted_root() {
.collect();
assert_eq!(expected.len(), 1);
for roots in [vec![root.clone()], vec![root.clone(), cwd.clone()]] {
- let executor =
- ReadExecutor::new(&cwd, ReadScope::Restricted { roots }, ReadLimits::default())
- .unwrap();
+ let executor = ReadExecutor::new(&cwd, ReadScope::Restricted { roots }).unwrap();
for path in [".", "src/../src"] {
let result = executor
.invoke(
diff --git a/plugins/workspace/workspace-files/engine/tests/policy.rs b/plugins/workspace/workspace-files/engine/tests/policy.rs
index 1cda21f9dd..822b4ae7e9 100644
--- a/plugins/workspace/workspace-files/engine/tests/policy.rs
+++ b/plugins/workspace/workspace-files/engine/tests/policy.rs
@@ -18,9 +18,7 @@
*/
use maka_agent_facts::tools::ToolExecutor;
-use maka_fs_tools::{
- MutationExecutor, ReadExecutor, ReadLimits, ReadScope, WriteCoordinator, WriteScope,
-};
+use maka_fs_tools::{MutationExecutor, ReadExecutor, ReadScope, WriteCoordinator, WriteScope};
use maka_sandbox::filesystem::{Access, Policy, Rule};
use serde_json::json;
use std::{fs, sync::Arc};
@@ -55,12 +53,7 @@ async fn read_search_and_mutations_share_nested_policy_including_missing_targets
.compile()
.unwrap(),
);
- let read = ReadExecutor::new(
- &root,
- ReadScope::Policy(policy.clone()),
- ReadLimits::default(),
- )
- .unwrap();
+ let read = ReadExecutor::new(&root, ReadScope::Policy(policy.clone())).unwrap();
let write = MutationExecutor::new(
&root,
WriteScope::Policy(policy),
diff --git a/plugins/workspace/workspace-files/engine/tests/read.rs b/plugins/workspace/workspace-files/engine/tests/read.rs
index 4b019b5d4c..963c3a1496 100644
--- a/plugins/workspace/workspace-files/engine/tests/read.rs
+++ b/plugins/workspace/workspace-files/engine/tests/read.rs
@@ -19,20 +19,19 @@
use maka_agent_facts::tools::ToolError;
use maka_agent_facts::tools::ToolExecutor;
-use maka_fs_tools::{ReadExecutor, ReadLimits, ReadScope};
+use maka_fs_tools::{ReadExecutor, ReadScope};
use serde_json::{Value, json};
use std::fs;
#[cfg(unix)]
use std::os::unix::fs::symlink;
use tokio_util::sync::CancellationToken;
-fn scoped(root: &std::path::Path, limits: ReadLimits) -> ReadExecutor {
+fn scoped(root: &std::path::Path) -> ReadExecutor {
ReadExecutor::new(
root,
ReadScope::Restricted {
roots: vec![root.to_owned()],
},
- limits,
)
.unwrap()
}
@@ -50,13 +49,7 @@ async fn file_pages_bind_the_full_source_and_preserve_limits_and_cancellation()
fs::write(root.join("text"), "零\r\n一\n二\n").unwrap();
fs::write(root.join("binary"), [0xff, 0xfe]).unwrap();
fs::write(root.join("nul"), b"a\0b").unwrap();
- fs::write(root.join("large"), vec![b'x'; 65]).unwrap();
- let executor = scoped(
- &root,
- ReadLimits {
- max_source_bytes: 64,
- },
- );
+ let executor = scoped(&root);
for (input, content, offset, returned) in [
(json!({"path":"text"}), "零\r\n一\n二\n", 0, 4),
(json!({"path":"text","offset":1,"limit":2}), "一\n二", 1, 2),
@@ -72,7 +65,6 @@ async fn file_pages_bind_the_full_source_and_preserve_limits_and_cancellation()
json!({"path":"runtime-resource://text"}),
json!({"path":"binary"}),
json!({"path":"nul"}),
- json!({"path":"large","limit":1}),
] {
assert!(read(&executor, input).await.is_err());
}
@@ -95,7 +87,7 @@ async fn file_pages_bind_the_full_source_and_preserve_limits_and_cancellation()
.is_err()
);
- let executor = scoped(&root, ReadLimits::default());
+ let executor = scoped(&root);
let line = "中文😀\\\"".repeat(3_000);
let source = format!("before\n{line}\nafter");
fs::write(root.join("pages"), &source).unwrap();
@@ -134,6 +126,43 @@ async fn file_pages_bind_the_full_source_and_preserve_limits_and_cancellation()
);
}
+#[tokio::test]
+async fn text_reads_validate_the_tail_and_carry_utf8_across_actual_io_cuts() {
+ let temp = tempfile::tempdir().unwrap();
+ let root = temp.path().canonicalize().unwrap();
+ let executor = scoped(&root);
+ for fill in [8180, 8181, 8182] {
+ let text = format!("\u{feff}first\n{}😀\nlast", "x".repeat(fill));
+ fs::write(root.join("carry"), &text).unwrap();
+ assert_eq!(
+ read(&executor, json!({"path":"carry","offset":0,"limit":1}))
+ .await
+ .unwrap(),
+ json!({"content":"\u{feff}first","offset":0,"returnedLines":1,"totalLines":3,"next":null})
+ );
+ assert_eq!(
+ read(&executor, json!({"path":"carry","offset":2,"limit":1}))
+ .await
+ .unwrap(),
+ json!({"content":"last","offset":2,"returnedLines":1,"totalLines":3,"next":null})
+ );
+ }
+ for tail in [&[0xff][..], &[0xe2, 0x82][..], &[0][..]] {
+ let mut bytes = b"first\n".to_vec();
+ bytes.resize(1_048_600, b'x');
+ bytes.extend_from_slice(tail);
+ fs::write(root.join("bad-tail"), bytes).unwrap();
+ let error = read(&executor, json!({"path":"bad-tail","offset":0,"limit":1}))
+ .await
+ .unwrap_err();
+ assert!(
+ matches!(error, ToolError::Failed(message) if message == if tail == [0] {
+ "Read does not support binary files"
+ } else { "Read requires UTF-8 text" })
+ );
+ }
+}
+
#[tokio::test]
#[cfg(unix)]
async fn captured_directory_survives_replacement_and_denies_escape_and_special_files() {
@@ -158,7 +187,6 @@ async fn captured_directory_survives_replacement_and_denies_escape_and_special_f
ReadScope::Restricted {
roots: vec![root.clone(), base.clone()],
},
- ReadLimits::default(),
)
.unwrap();
let outside_cursor = read(&overlapping, json!({"path":"../outside/pages"}))
@@ -184,7 +212,6 @@ async fn captured_directory_survives_replacement_and_denies_escape_and_special_f
ReadScope::Restricted {
roots: vec![root.clone(), alias.clone()],
},
- ReadLimits::default(),
)
.unwrap();
assert!(read(&executor, outside_cursor).await.is_err());
@@ -256,8 +283,7 @@ async fn captured_directory_survives_replacement_and_denies_escape_and_special_f
.await
.expect("opening a FIFO must not wait for a writer")
.unwrap_err();
- let bypass =
- ReadExecutor::new(&outside, ReadScope::Unrestricted, ReadLimits::default()).unwrap();
+ let bypass = ReadExecutor::new(&outside, ReadScope::Unrestricted).unwrap();
assert_eq!(
read(&bypass, json!({"path":outside.join("text")}))
.await
diff --git a/plugins/workspace/workspace-files/engine/tests/windows_read.rs b/plugins/workspace/workspace-files/engine/tests/windows_read.rs
index 98a6eac565..56df8d0986 100644
--- a/plugins/workspace/workspace-files/engine/tests/windows_read.rs
+++ b/plugins/workspace/workspace-files/engine/tests/windows_read.rs
@@ -20,9 +20,7 @@
#![cfg(windows)]
use maka_agent_facts::tools::ToolExecutor;
-use maka_fs_tools::{
- MutationExecutor, ReadExecutor, ReadLimits, ReadScope, WriteCoordinator, WriteScope,
-};
+use maka_fs_tools::{MutationExecutor, ReadExecutor, ReadScope, WriteCoordinator, WriteScope};
use serde_json::json;
use std::{fs, path::Path, sync::Arc};
use tokio_util::sync::CancellationToken;
@@ -80,7 +78,6 @@ async fn case_sensitive_sibling_roots_do_not_alias_reads_or_mutations() {
ReadScope::Restricted {
roots: vec![upper.clone()],
},
- ReadLimits::default(),
)
.unwrap();
assert!(read(&reader, lower.join("file")).await.is_err());
@@ -147,7 +144,6 @@ async fn windows_path_spellings_preserve_the_captured_scope_and_directory_identi
ReadScope::Restricted {
roots: vec![canonical.clone()],
},
- ReadLimits::default(),
)
.unwrap();
for path in [
@@ -188,7 +184,6 @@ async fn windows_path_spellings_preserve_the_captured_scope_and_directory_identi
ReadScope::Restricted {
roots: vec![canonical.clone(), temp.path().to_owned()],
},
- ReadLimits::default(),
)
.unwrap();
assert_eq!(
@@ -207,8 +202,7 @@ async fn windows_path_spellings_preserve_the_captured_scope_and_directory_identi
drop(union);
fs::rename(&root, &moved).unwrap();
- let bypass =
- ReadExecutor::new(&outside, ReadScope::Unrestricted, ReadLimits::default()).unwrap();
+ let bypass = ReadExecutor::new(&outside, ReadScope::Unrestricted).unwrap();
assert_eq!(
read(&bypass, outside.join("text")).await.unwrap(),
json!({"content":"outside","offset":0,"returnedLines":1,"totalLines":1,"next":null})
diff --git a/plugins/workspace/workspace-files/native/src/attachments.rs b/plugins/workspace/workspace-files/native/src/attachments.rs
index df7706948e..9351ed3736 100644
--- a/plugins/workspace/workspace-files/native/src/attachments.rs
+++ b/plugins/workspace/workspace-files/native/src/attachments.rs
@@ -85,15 +85,9 @@ pub async fn read(
if text.contains('\0') {
return Err(failed("Read cannot decode this binary attachment as text"));
}
- // Reserve the public service envelope as well as the ReadPage itself;
- // otherwise pruning hides the attachment continuation inside an archive.
- let envelope = serde_json::to_string(&FileResult::Completed { value: Value::Null })
- .expect("file result envelope")
- .len()
- - "null".len();
serde_json::to_value(
request
- .page_with_budget(&text, maka_agent_facts::read::MAX_PAGE_CHARS - envelope)
+ .page_with_budget(&text, read_page_budget())
.map_err(failed)?,
)
.map(Output::Json)
diff --git a/plugins/workspace/workspace-files/native/src/lib.rs b/plugins/workspace/workspace-files/native/src/lib.rs
index 6f75a0d22b..4c4f6eb61b 100644
--- a/plugins/workspace/workspace-files/native/src/lib.rs
+++ b/plugins/workspace/workspace-files/native/src/lib.rs
@@ -27,7 +27,7 @@ use maka_agent_facts::tools::ToolError;
use maka_agent_facts::tools::ToolExecutor;
use maka_event_log::EventLog;
use maka_fs_tools::{
- MutationExecutor, ReadExecutor, ReadLimits, ReadOutput, ReadScope, WriteCoordinator, WriteScope,
+ MutationExecutor, ReadExecutor, ReadOutput, ReadScope, WriteCoordinator, WriteScope,
};
use maka_host_runtime::application::{
Context, Handler, NativeActivation, NativeOwner, NativeRegistry,
@@ -359,24 +359,19 @@ async fn execute(
.map(Output::Json)
} else {
let executor = if let Some(policy) = policy {
- ReadExecutor::from_directory(
- captured.path,
- captured.directory,
- ReadLimits::default(),
- Some(policy),
- )?
+ ReadExecutor::from_directory(captured.path, captured.directory, Some(policy))?
} else {
- ReadExecutor::new(
- captured.path,
- ReadScope::Unrestricted,
- ReadLimits::default(),
- )?
+ ReadExecutor::new(captured.path, ReadScope::Unrestricted)?
};
if tool == maka_fs_tools::READ_NAME {
let input: maka_agent_facts::read::ReadInput =
serde_json::from_value(input).map_err(failed)?;
match executor
- .read(input.resolve().map_err(failed)?, cancellation.clone())
+ .read(
+ input.resolve().map_err(failed)?,
+ read_page_budget(),
+ cancellation.clone(),
+ )
.await?
{
ReadOutput::Text(_) if image => {
@@ -410,3 +405,12 @@ fn cancelled(cancellation: &CancellationToken) -> Result<(), ToolError> {
fn failed(message: impl std::fmt::Display) -> ToolError {
ToolError::Failed(message.to_string())
}
+
+// Reserve the public service envelope so model pruning preserves the file continuation.
+fn read_page_budget() -> usize {
+ let envelope = serde_json::to_string(&FileResult::Completed { value: Value::Null })
+ .expect("file result envelope")
+ .len()
+ - "null".len();
+ maka_agent_facts::read::MAX_PAGE_CHARS - envelope
+}
diff --git a/products/maka/cli/src/code.rs b/products/maka/cli/src/code.rs
index 00a8256b1c..ab9f9a1219 100644
--- a/products/maka/cli/src/code.rs
+++ b/products/maka/cli/src/code.rs
@@ -37,9 +37,7 @@ use maka_agent_facts::tools::ToolError;
use maka_agent_facts::tools::ToolExecutor;
use maka_agent_facts::tools::ToolFuture;
use maka_event_log::EventLog;
-use maka_fs_tools::{
- MutationExecutor, ReadExecutor, ReadLimits, ReadScope, WriteCoordinator, WriteScope,
-};
+use maka_fs_tools::{MutationExecutor, ReadExecutor, ReadScope, WriteCoordinator, WriteScope};
use maka_js_runtime::{
CellAbort, CellContext, CellDiagnosticKind, CellLimits, CellResult, CellStore, CodeExecutor,
ToolMetadata,
@@ -149,7 +147,7 @@ pub(super) async fn run(args: Args) -> Result<(), maka_host_runtime::HostError>
Sandbox::External { .. } => unreachable!("workspace presets never delegate isolation"),
};
let local_tools = LocalTools {
- read: ReadExecutor::new(&cwd, read_scope, ReadLimits::default())?,
+ read: ReadExecutor::new(&cwd, read_scope)?,
write: MutationExecutor::new(&cwd, write_scope, Arc::new(WriteCoordinator::default()))?,
};
let limits = CellLimits::default();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@maka.apache.org
For additional commands, e-mail: commits-help@maka.apache.org