You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/06/18 08:08:12 UTC
(opendal) 01/01: test: mock hf read and list contracts
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/hf-mock-contract-tests
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit d817b810fa8f815ac643bb3f15cc82346d5405ac
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Thu Jun 18 16:07:51 2026 +0800
test: mock hf read and list contracts
---
core/services/hf/src/backend.rs | 11 -----------
core/services/hf/src/lister.rs | 39 +++++++++++++++++++++++++++++++++------
core/services/hf/src/reader.rs | 27 +++++++++++++++++++++------
3 files changed, 54 insertions(+), 23 deletions(-)
diff --git a/core/services/hf/src/backend.rs b/core/services/hf/src/backend.rs
index f4fccdb50..69cd85dce 100644
--- a/core/services/hf/src/backend.rs
+++ b/core/services/hf/src/backend.rs
@@ -426,17 +426,6 @@ pub(super) mod test_utils {
op.with_context(OperationContext::new().with_http_transport(transport))
}
- pub fn gpt2_operator() -> Operator {
- let op = Operator::new(
- HfBuilder::default()
- .repo_type("model")
- .repo_id("openai-community/gpt2")
- .download_mode("http"),
- )
- .unwrap();
- finish_operator(op)
- }
-
pub fn mbpp_operator() -> Operator {
let op = Operator::new(
HfBuilder::default()
diff --git a/core/services/hf/src/lister.rs b/core/services/hf/src/lister.rs
index d8133ca7e..deade36cd 100644
--- a/core/services/hf/src/lister.rs
+++ b/core/services/hf/src/lister.rs
@@ -159,8 +159,13 @@ impl oio::PageList for HfLister {
#[cfg(test)]
mod tests {
- use super::super::backend::test_utils::gpt2_operator;
+ use std::collections::VecDeque;
+ use std::sync::Arc;
+
+ use super::super::core::test_utils::create_test_core;
+ use super::super::uri::HfRepoType;
use super::*;
+ use opendal_core::raw::oio::PageList;
#[test]
fn test_parse_next_cursor() {
@@ -177,10 +182,32 @@ mod tests {
}
#[tokio::test]
- async fn test_list_model_root() {
- let op = gpt2_operator();
- let entries = op.list("/").await.expect("list should succeed");
- let names: Vec<&str> = entries.iter().map(|e| e.name()).collect();
- assert!(names.contains(&"config.json"));
+ async fn test_list_model_root() -> Result<()> {
+ let (core, ctx, mock_client) = create_test_core(
+ HfRepoType::Model,
+ "test-user/test-repo",
+ "main",
+ "https://huggingface.co",
+ );
+ let lister = HfLister::new(Arc::new(core), ctx, String::new(), false);
+ let mut page_ctx = oio::PageContext {
+ done: false,
+ token: String::new(),
+ entries: VecDeque::new(),
+ };
+
+ lister.next_page(&mut page_ctx).await?;
+
+ assert_eq!(
+ mock_client.get_captured_url(),
+ "https://huggingface.co/api/models/test-user/test-repo/tree/main/?expand=True"
+ );
+ assert!(page_ctx.done);
+ let entry = page_ctx.entries.pop_front().expect("entry must exist");
+ assert_eq!(entry.path(), "test.txt");
+ assert_eq!(entry.metadata().mode(), EntryMode::FILE);
+ assert_eq!(entry.metadata().content_length(), 100);
+
+ Ok(())
}
}
diff --git a/core/services/hf/src/reader.rs b/core/services/hf/src/reader.rs
index e51f13cd3..7705b8c8a 100644
--- a/core/services/hf/src/reader.rs
+++ b/core/services/hf/src/reader.rs
@@ -100,7 +100,7 @@ impl oio::ReadStream for HfReadStream {
#[cfg(test)]
mod tests {
- use super::super::backend::test_utils::{gpt2_operator, mbpp_operator, testing_dataset_core};
+ use super::super::backend::test_utils::{mbpp_operator, testing_dataset_core};
use super::super::core::test_utils::create_test_core;
use super::super::core::{CommitFile, DeletedFile};
use super::super::uri::HfRepoType;
@@ -112,11 +112,26 @@ mod tests {
const PARQUET_MAGIC: &[u8] = b"PAR1";
#[tokio::test]
- async fn test_read_model_config() {
- let op = gpt2_operator();
- let data = op.read("config.json").await.expect("read should succeed");
- serde_json::from_slice::<serde_json::Value>(&data.to_vec())
- .expect("config.json should be valid JSON");
+ async fn test_http_read_uses_resolve_url() -> Result<()> {
+ let (core, ctx, mock_client) = create_test_core(
+ HfRepoType::Model,
+ "test-user/test-repo",
+ "main",
+ "https://huggingface.co",
+ );
+
+ let (_, mut reader) =
+ HfReadStream::try_new(&core, &ctx, "config.json", BytesRange::default()).await?;
+
+ assert_eq!(
+ mock_client.get_captured_url(),
+ "https://huggingface.co/test-user/test-repo/resolve/main/config.json"
+ );
+ assert!(matches!(reader, HfReadStream::Http(_)));
+ let chunk = reader.read().await?;
+ assert_eq!(chunk.to_bytes(), Bytes::from_static(b"hello"));
+
+ Ok(())
}
#[tokio::test]