You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hugegraph.apache.org by ji...@apache.org on 2026/05/31 01:27:12 UTC
(hugegraph-ai) 04/13: test(quality): define test taxonomy and baseline
This is an automated email from the ASF dual-hosted git repository.
imbajin pushed a commit to branch goal-test
in repository https://gitbox.apache.org/repos/asf/hugegraph-ai.git
commit 0659cae999ed83216b9b3a171c0bd79265d2801a
Author: imbajin <ji...@apache.org>
AuthorDate: Sun May 31 01:27:19 2026 +0800
test(quality): define test taxonomy and baseline
- add strict pytest markers for quality layers
- mark existing deterministic and integration tests
- generate initial client and llm coverage baselines
- document taxonomy, skips, and baseline status
---
.workflow/quality-program/baseline.md | 34 +++++++++++
.../quality-program/checkpoints/01-taxonomy.md | 66 ++++++++++++++++++++++
.../quality-program/coverage/client-baseline.json | 1 +
.../quality-program/coverage/llm-baseline.json | 1 +
.workflow/quality-program/quality-state.json | 65 ++++++++++++++++++---
docs/quality/test-taxonomy.md | 32 +++++++++++
.../2026-05-31-hugegraph-ai-quality-program.md | 16 +++---
hugegraph-llm/src/tests/api/test_rag_api.py | 3 +
hugegraph-llm/src/tests/config/test_config.py | 4 ++
.../src/tests/config/test_prompt_config.py | 4 ++
hugegraph-llm/src/tests/conftest.py | 5 +-
hugegraph-llm/src/tests/document/test_document.py | 4 ++
.../src/tests/document/test_document_splitter.py | 4 ++
.../src/tests/document/test_text_loader.py | 4 ++
.../src/tests/middleware/test_middleware.py | 3 +
.../src/tests/models/llms/test_litellm_client.py | 3 +
.../src/tests/models/llms/test_openai_client.py | 4 ++
.../operators/llm_op/test_disambiguate_data.py | 4 ++
.../operators/llm_op/test_gremlin_generate.py | 4 ++
.../tests/operators/llm_op/test_info_extract.py | 4 ++
.../tests/operators/llm_op/test_keyword_extract.py | 4 ++
.../llm_op/test_property_graph_extract.py | 4 ++
hugegraph-python-client/src/tests/api/test_auth.py | 3 +
.../src/tests/api/test_auth_routing.py | 2 +
.../src/tests/api/test_graph.py | 3 +
.../src/tests/api/test_graphs.py | 4 ++
.../src/tests/api/test_gremlin.py | 2 +
.../src/tests/api/test_metric.py | 4 ++
.../src/tests/api/test_response_validation.py | 3 +
.../src/tests/api/test_schema.py | 4 ++
hugegraph-python-client/src/tests/api/test_task.py | 3 +
.../src/tests/api/test_traverser.py | 4 ++
.../src/tests/api/test_variable.py | 2 +
.../src/tests/api/test_version.py | 4 ++
pyproject.toml | 12 ++++
35 files changed, 306 insertions(+), 17 deletions(-)
diff --git a/.workflow/quality-program/baseline.md b/.workflow/quality-program/baseline.md
new file mode 100644
index 00000000..06d7e848
--- /dev/null
+++ b/.workflow/quality-program/baseline.md
@@ -0,0 +1,34 @@
+# Quality Program Baseline
+
+## Test Selection Baseline
+
+| Module | Command | Result |
+|---|---|---|
+| `hugegraph-python-client` all tests collection | `uv run pytest hugegraph-python-client/src/tests --collect-only -q` | 71 collected |
+| `hugegraph-python-client` Layer A collection | `uv run pytest hugegraph-python-client/src/tests -m "unit or contract" --collect-only -q` | 15 selected, 56 deselected |
+| `hugegraph-llm` all tests collection from repo root | `uv run pytest hugegraph-llm/src/tests --collect-only -q` | failed before collection due existing prompt-config cwd guard |
+| `hugegraph-llm` Layer A collection from repo root | `uv run pytest hugegraph-llm/src/tests -m "unit or contract" --collect-only -q` | failed before collection due existing prompt-config cwd guard |
+| `hugegraph-llm` all tests collection from module cwd | `uv run pytest src/tests --collect-only -q` | 276 collected |
+| `hugegraph-llm` Layer A collection from module cwd | `uv run pytest src/tests -m "unit or contract" --collect-only -q` | 101 selected, 175 deselected |
+
+## Coverage Baseline
+
+| Module | Command | Result | Artifact |
+|---|---|---:|---|
+| `pyhugegraph` | `uv run pytest hugegraph-python-client/src/tests -m "unit or contract" --cov=pyhugegraph --cov-report=term --cov-report=json:.workflow/quality-program/coverage/client-baseline.json` | 45% | `.workflow/quality-program/coverage/client-baseline.json` |
+| `hugegraph_llm` | `uv run pytest src/tests -m "unit or contract" --cov=hugegraph_llm --cov-report=term --cov-report=json:../.workflow/quality-program/coverage/llm-baseline.json` from `hugegraph-llm/` | 34% | `.workflow/quality-program/coverage/llm-baseline.json` |
+
+## Existing Skips and Service Controls
+
+- `hugegraph-llm/src/tests/conftest.py` now uses `os.environ.setdefault("SKIP_EXTERNAL_SERVICES", "true")`, so selected integration runs can opt in explicitly.
+- Client Gremlin tests still support explicit `SKIP_GREMLIN_TESTS=true`.
+- LLM provider and external-service tests remain outside the initial `unit or contract` baseline unless they are deterministic mocked contracts.
+- Existing `hugegraph-llm` imports can require the current working directory to be `hugegraph-llm`; this is recorded as a baseline command mismatch for later cleanup, not fixed in G0.
+
+## Weak or Unclassified Integration Tests
+
+- `hugegraph-llm/src/tests/integration/test_graph_rag_pipeline.py`
+- `hugegraph-llm/src/tests/integration/test_kg_construction.py`
+- `hugegraph-llm/src/tests/integration/test_rag_pipeline.py`
+
+These are not reclassified in G0. G5 must inspect whether they import production code, define local replacement pipelines, or only assert mock calls.
diff --git a/.workflow/quality-program/checkpoints/01-taxonomy.md b/.workflow/quality-program/checkpoints/01-taxonomy.md
new file mode 100644
index 00000000..e135a3f3
--- /dev/null
+++ b/.workflow/quality-program/checkpoints/01-taxonomy.md
@@ -0,0 +1,66 @@
+# G0 Taxonomy Checkpoint
+
+## Files Touched
+
+- `pyproject.toml`
+- `docs/quality/test-taxonomy.md`
+- `.workflow/quality-program/baseline.md`
+- `.workflow/quality-program/checkpoints/01-taxonomy.md`
+- `.workflow/quality-program/coverage/client-baseline.json`
+- `.workflow/quality-program/coverage/llm-baseline.json`
+- Selected deterministic tests under `hugegraph-llm/src/tests/`
+- Existing service-bound client tests under `hugegraph-python-client/src/tests/api/`
+
+## Markers Added
+
+- Root pytest markers: `unit`, `contract`, `integration`, `hugegraph`, `smoke`, `external`, `slow`.
+- Client deterministic contract tests:
+ - `test_auth_routing.py`
+ - `test_response_validation.py`
+- Client service-bound tests:
+ - `test_auth.py`
+ - `test_graph.py`
+ - `test_graphs.py`
+ - `test_gremlin.py`
+ - `test_metric.py`
+ - `test_schema.py`
+ - `test_task.py`
+ - `test_traverser.py`
+ - `test_variable.py`
+ - `test_version.py`
+- LLM deterministic `unit` or `contract` tests:
+ - API config route contract test
+ - config and prompt config tests
+ - document and middleware tests
+ - selected mocked OpenAI/LiteLLM tests
+ - selected LLM operator parser/generation tests
+
+## Commands Run
+
+```bash
+uv run pytest hugegraph-python-client/src/tests --collect-only -q
+uv run pytest hugegraph-llm/src/tests --collect-only -q
+uv run pytest hugegraph-python-client/src/tests -m "unit or contract" --collect-only -q
+uv run pytest hugegraph-llm/src/tests -m "unit or contract" --collect-only -q
+uv run pytest src/tests --collect-only -q
+uv run pytest src/tests -m "unit or contract" --collect-only -q
+uv sync --extra dev --extra python-client --extra llm
+uv run pytest hugegraph-python-client/src/tests -m "unit or contract" --cov=pyhugegraph --cov-report=term --cov-report=json:.workflow/quality-program/coverage/client-baseline.json
+uv run pytest src/tests -m "unit or contract" --cov=hugegraph_llm --cov-report=term --cov-report=json:../.workflow/quality-program/coverage/llm-baseline.json
+```
+
+## Coverage Baseline
+
+- `pyhugegraph`: 45% line coverage, 15 selected contract tests passed.
+- `hugegraph_llm`: 34% line coverage, 101 selected unit/contract tests passed.
+
+## Failures or Skips Observed
+
+- Root-level LLM collection command failed before collection because existing prompt config requires current directory `hugegraph-llm`.
+- First coverage attempt failed because `pytest-cov` was not installed in the active `.venv`; `uv sync --extra dev --extra python-client --extra llm` resolved it.
+- No unknown marker errors after strict marker config was added.
+
+## Next Goal Readiness
+
+- Ready for G1 service fixture and CI standardization.
+- G1 should keep Layer A tests Docker-free and make selected Layer B failures explicit under `HUGEGRAPH_REQUIRED=true`.
diff --git a/.workflow/quality-program/coverage/client-baseline.json b/.workflow/quality-program/coverage/client-baseline.json
new file mode 100644
index 00000000..430fb03b
--- /dev/null
+++ b/.workflow/quality-program/coverage/client-baseline.json
@@ -0,0 +1 @@
+{"meta": {"format": 3, "version": "7.10.4", "timestamp": "2026-05-31T01:24:32.655986", "branch_coverage": false, "show_contexts": false}, "files": {"hugegraph-python-client/src/pyhugegraph/__init__.py": {"executed_lines": [1], "summary": {"covered_lines": 0, "num_statements": 0, "percent_covered": 100.0, "percent_covered_display": "100", "missing_lines": 0, "excluded_lines": 0}, "missing_lines": [], "excluded_lines": [], "functions": {"": {"executed_lines": [1], "summary": {"covered_line [...]
diff --git a/.workflow/quality-program/coverage/llm-baseline.json b/.workflow/quality-program/coverage/llm-baseline.json
new file mode 100644
index 00000000..bda3a652
--- /dev/null
+++ b/.workflow/quality-program/coverage/llm-baseline.json
@@ -0,0 +1 @@
+{"meta": {"format": 3, "version": "7.10.4", "timestamp": "2026-05-31T01:24:44.683144", "branch_coverage": false, "show_contexts": false}, "files": {"src/hugegraph_llm/__init__.py": {"executed_lines": [1], "summary": {"covered_lines": 0, "num_statements": 0, "percent_covered": 100.0, "percent_covered_display": "100", "missing_lines": 0, "excluded_lines": 0}, "missing_lines": [], "excluded_lines": [], "functions": {"": {"executed_lines": [1], "summary": {"covered_lines": 0, "num_statements [...]
diff --git a/.workflow/quality-program/quality-state.json b/.workflow/quality-program/quality-state.json
index d8de5ca4..3838517e 100644
--- a/.workflow/quality-program/quality-state.json
+++ b/.workflow/quality-program/quality-state.json
@@ -1,10 +1,11 @@
{
- "current_goal": "G0",
+ "current_goal": "G1",
"repo_sha_start": "d048bc9fa07042833549dd1def5ff3222c47be70",
"base_branch": "main",
"open_pr_snapshot_time": "2026-05-30T17:18:30Z",
"goals_completed": [
- "P0"
+ "P0",
+ "G0"
],
"files_touched": [
".workflow/quality-program/README.md",
@@ -12,10 +13,45 @@
".workflow/quality-program/checkpoints/00-preflight.md",
".workflow/quality-program/reports/test-matrix.md",
".workflow/quality-program/reports/service-matrix.md",
- ".workflow/quality-program/reports/flaky-risk-ledger.md"
+ ".workflow/quality-program/reports/flaky-risk-ledger.md",
+ "pyproject.toml",
+ "docs/quality/test-taxonomy.md",
+ ".workflow/quality-program/baseline.md",
+ ".workflow/quality-program/checkpoints/01-taxonomy.md",
+ ".workflow/quality-program/coverage/client-baseline.json",
+ ".workflow/quality-program/coverage/llm-baseline.json",
+ "hugegraph-llm/src/tests/conftest.py",
+ "hugegraph-llm/src/tests/api/test_rag_api.py",
+ "hugegraph-llm/src/tests/config/test_config.py",
+ "hugegraph-llm/src/tests/config/test_prompt_config.py",
+ "hugegraph-llm/src/tests/document/test_document.py",
+ "hugegraph-llm/src/tests/document/test_document_splitter.py",
+ "hugegraph-llm/src/tests/document/test_text_loader.py",
+ "hugegraph-llm/src/tests/middleware/test_middleware.py",
+ "hugegraph-llm/src/tests/models/llms/test_litellm_client.py",
+ "hugegraph-llm/src/tests/models/llms/test_openai_client.py",
+ "hugegraph-llm/src/tests/operators/llm_op/test_disambiguate_data.py",
+ "hugegraph-llm/src/tests/operators/llm_op/test_gremlin_generate.py",
+ "hugegraph-llm/src/tests/operators/llm_op/test_info_extract.py",
+ "hugegraph-llm/src/tests/operators/llm_op/test_keyword_extract.py",
+ "hugegraph-llm/src/tests/operators/llm_op/test_property_graph_extract.py",
+ "hugegraph-python-client/src/tests/api/test_auth.py",
+ "hugegraph-python-client/src/tests/api/test_auth_routing.py",
+ "hugegraph-python-client/src/tests/api/test_graph.py",
+ "hugegraph-python-client/src/tests/api/test_graphs.py",
+ "hugegraph-python-client/src/tests/api/test_gremlin.py",
+ "hugegraph-python-client/src/tests/api/test_metric.py",
+ "hugegraph-python-client/src/tests/api/test_response_validation.py",
+ "hugegraph-python-client/src/tests/api/test_schema.py",
+ "hugegraph-python-client/src/tests/api/test_task.py",
+ "hugegraph-python-client/src/tests/api/test_traverser.py",
+ "hugegraph-python-client/src/tests/api/test_variable.py",
+ "hugegraph-python-client/src/tests/api/test_version.py"
],
"production_changes": [],
- "tests_added_or_changed": [],
+ "tests_added_or_changed": [
+ "Added file-level unit/contract/integration/hugegraph markers to selected existing tests for Layer A and Layer B selection."
+ ],
"commands_run": [
"sed -n '1,220p' AGENTS.md",
"sed -n '1,260p' hugegraph-llm/AGENTS.md",
@@ -29,9 +65,24 @@
"git diff -- . ':!/.workflow/quality-program'",
"git status --short",
"git add .workflow/quality-program docs/superpowers/plans/2026-05-31-hugegraph-ai-quality-program.md",
- "git commit -m \"docs(quality): add quality program preflight ledger\" -m \"- initialize restartable campaign state and checkpoints\n- document current CI and service matrix scaffolds\n- capture PR collision and skip-control audit structure\""
+ "git commit -m \"docs(quality): add quality program preflight ledger\" -m \"- initialize restartable campaign state and checkpoints\n- document current CI and service matrix scaffolds\n- capture PR collision and skip-control audit structure\"",
+ "uv run pytest hugegraph-python-client/src/tests --collect-only -q",
+ "uv run pytest hugegraph-llm/src/tests --collect-only -q",
+ "uv run pytest hugegraph-python-client/src/tests -m \"unit or contract\" --collect-only -q",
+ "uv run pytest hugegraph-llm/src/tests -m \"unit or contract\" --collect-only -q",
+ "uv run pytest src/tests --collect-only -q",
+ "uv run pytest src/tests -m \"unit or contract\" --collect-only -q",
+ "uv sync --extra dev --extra python-client --extra llm",
+ "uv run pytest hugegraph-python-client/src/tests -m \"unit or contract\" --cov=pyhugegraph --cov-report=term --cov-report=json:.workflow/quality-program/coverage/client-baseline.json",
+ "uv run pytest src/tests -m \"unit or contract\" --cov=hugegraph_llm --cov-report=term --cov-report=json:../.workflow/quality-program/coverage/llm-baseline.json",
+ "uv run ruff format --check hugegraph-llm/src/tests hugegraph-python-client/src/tests pyproject.toml",
+ "uv run ruff check hugegraph-llm/src/tests hugegraph-python-client/src/tests pyproject.toml",
+ "uv run ruff check hugegraph-python-client/src/tests/api/test_auth.py hugegraph-python-client/src/tests/api/test_graph.py hugegraph-python-client/src/tests/api/test_task.py --fix"
+ ],
+ "known_failures": [
+ "Root-level LLM collection command fails before collection because existing prompt config requires cwd hugegraph-llm.",
+ "Initial coverage command failed before uv sync because pytest-cov was missing from the active environment."
],
- "known_failures": [],
"deferred_items": [
"Avoid #350/#277 YAML config migration files unless tests require a narrow compatibility assertion.",
"Avoid #179 async/streaming API refactor surface.",
@@ -39,5 +90,5 @@
"Avoid #240 vector/property embedding abstraction surface.",
"Quarantine client/test fixes overlapping #323/#329/#342 until exact changed files are inspected."
],
- "next_recommended_action": "Start G0 baseline and test taxonomy"
+ "next_recommended_action": "Standardize HugeGraph service fixture and CI readiness"
}
diff --git a/docs/quality/test-taxonomy.md b/docs/quality/test-taxonomy.md
new file mode 100644
index 00000000..a93ecb42
--- /dev/null
+++ b/docs/quality/test-taxonomy.md
@@ -0,0 +1,32 @@
+# HugeGraph AI Test Taxonomy
+
+## Layer A: Unit / Pure Contract
+
+- Markers: `unit` or `contract`.
+- No Docker, network, real HugeGraph, real LLM provider, embedding provider, reranker provider, vector DB, or UI service.
+- Use fakes, fixtures, monkeypatches, and public APIs.
+
+## Layer B: HugeGraph Server Contract
+
+- Markers: `integration` and `hugegraph`.
+- Requires HugeGraph Server, default `hugegraph/hugegraph:1.7.0`.
+- If selected with `HUGEGRAPH_REQUIRED=true`, service connection failures fail.
+- Must import production code and validate real server behavior.
+
+## Layer C: Core Pipeline Smoke
+
+- Marker: `smoke`; also use `integration` and `hugegraph` when real HugeGraph is required.
+- Uses production flow/node/operator code.
+- Uses fake LLM and deterministic embeddings/vector fixtures.
+- Does not define local replacement pipeline implementations inside tests.
+
+## Layer D: External Provider / Optional E2E
+
+- Markers: `external` and usually `slow`.
+- May require real provider credentials or non-HugeGraph services.
+- Excluded from default PR gates.
+
+## Required Skip Semantics
+
+Do not silently skip selected Layer B tests. Prefer not selecting integration tests locally.
+If `HUGEGRAPH_REQUIRED=true`, unavailable HugeGraph is a failure.
diff --git a/docs/superpowers/plans/2026-05-31-hugegraph-ai-quality-program.md b/docs/superpowers/plans/2026-05-31-hugegraph-ai-quality-program.md
index 20c7c6d8..770d356d 100644
--- a/docs/superpowers/plans/2026-05-31-hugegraph-ai-quality-program.md
+++ b/docs/superpowers/plans/2026-05-31-hugegraph-ai-quality-program.md
@@ -231,7 +231,7 @@ git commit -m "docs(quality): add quality program preflight ledger" -m "- initia
- Create/Update: `.workflow/quality-program/coverage/client-baseline.json`
- Create/Update: `.workflow/quality-program/coverage/llm-baseline.json`
-- [ ] **Step G0.1: Add strict pytest marker definitions**
+- [x] **Step G0.1: Add strict pytest marker definitions**
Modify root `pyproject.toml` by adding this section if no `[tool.pytest.ini_options]` exists:
@@ -251,7 +251,7 @@ addopts = "--strict-markers --strict-config"
If `[tool.pytest.ini_options]` exists, merge the marker list without removing existing options.
-- [ ] **Step G0.2: Create test taxonomy documentation**
+- [x] **Step G0.2: Create test taxonomy documentation**
Create `docs/quality/test-taxonomy.md`:
@@ -290,7 +290,7 @@ Do not silently skip selected Layer B tests. Prefer not selecting integration te
If `HUGEGRAPH_REQUIRED=true`, unavailable HugeGraph is a failure.
```
-- [ ] **Step G0.3: Mark current tests in small batches**
+- [x] **Step G0.3: Mark current tests in small batches**
Add `pytestmark = pytest.mark.unit` or `pytestmark = pytest.mark.contract` to existing deterministic tests that do not require Docker or network.
@@ -323,7 +323,7 @@ import pytest
pytestmark = [pytest.mark.integration, pytest.mark.hugegraph]
```
-- [ ] **Step G0.4: Remove global forced external skip from LLM conftest**
+- [x] **Step G0.4: Remove global forced external skip from LLM conftest**
Modify `hugegraph-llm/src/tests/conftest.py` so it does not always set `SKIP_EXTERNAL_SERVICES=true`.
@@ -341,7 +341,7 @@ os.environ.setdefault("SKIP_EXTERNAL_SERVICES", "true")
Rationale: Layer A remains safe by default, while integration runs can override the variable.
-- [ ] **Step G0.5: Run marker collection checks**
+- [x] **Step G0.5: Run marker collection checks**
Run:
@@ -354,7 +354,7 @@ uv run pytest hugegraph-llm/src/tests -m "unit or contract" --collect-only -q
Expected: no unknown marker errors. If a selected set is empty, mark more existing deterministic tests before continuing.
-- [ ] **Step G0.6: Generate baseline coverage artifacts**
+- [x] **Step G0.6: Generate baseline coverage artifacts**
Run:
@@ -366,7 +366,7 @@ uv run pytest hugegraph-llm/src/tests -m "unit or contract" --cov=hugegraph_llm
Expected: coverage JSON files exist. If legacy failures appear, record them in `.workflow/quality-program/baseline.md` and continue only if they are unrelated to marker setup.
-- [ ] **Step G0.7: Write taxonomy checkpoint**
+- [x] **Step G0.7: Write taxonomy checkpoint**
Create `.workflow/quality-program/checkpoints/01-taxonomy.md`:
@@ -397,7 +397,7 @@ Update `quality-state.json`:
Preserve existing arrays and append touched files, tests, and commands.
-- [ ] **Step G0.8: Commit G0**
+- [x] **Step G0.8: Commit G0**
Run:
diff --git a/hugegraph-llm/src/tests/api/test_rag_api.py b/hugegraph-llm/src/tests/api/test_rag_api.py
index 55fbd679..789da403 100644
--- a/hugegraph-llm/src/tests/api/test_rag_api.py
+++ b/hugegraph-llm/src/tests/api/test_rag_api.py
@@ -17,11 +17,14 @@
from unittest.mock import Mock
+import pytest
from fastapi import APIRouter, FastAPI, status
from fastapi.testclient import TestClient
from hugegraph_llm.api.rag_api import rag_http_api
+pytestmark = pytest.mark.contract
+
def test_graph_config_api_passes_graph_field_to_apply_graph_conf():
apply_graph_conf = Mock(return_value=status.HTTP_200_OK)
diff --git a/hugegraph-llm/src/tests/config/test_config.py b/hugegraph-llm/src/tests/config/test_config.py
index 63dc6cb9..9a2abf66 100644
--- a/hugegraph-llm/src/tests/config/test_config.py
+++ b/hugegraph-llm/src/tests/config/test_config.py
@@ -18,6 +18,10 @@
import unittest
+import pytest
+
+pytestmark = pytest.mark.unit
+
class TestConfig(unittest.TestCase):
def test_config(self):
diff --git a/hugegraph-llm/src/tests/config/test_prompt_config.py b/hugegraph-llm/src/tests/config/test_prompt_config.py
index 0a46b4c6..c300a551 100644
--- a/hugegraph-llm/src/tests/config/test_prompt_config.py
+++ b/hugegraph-llm/src/tests/config/test_prompt_config.py
@@ -19,10 +19,14 @@ import json
from pathlib import Path
from unittest.mock import MagicMock
+import pytest
+
from hugegraph_llm.config.prompt_config import PromptConfig
from hugegraph_llm.models.llms.base import BaseLLM
from hugegraph_llm.operators.llm_op.property_graph_extract import PropertyGraphExtract
+pytestmark = pytest.mark.contract
+
def _json_objects_after_marker(prompt, marker):
start = prompt.index(marker) + len(marker)
diff --git a/hugegraph-llm/src/tests/conftest.py b/hugegraph-llm/src/tests/conftest.py
index ff38f3d5..8819e515 100644
--- a/hugegraph-llm/src/tests/conftest.py
+++ b/hugegraph-llm/src/tests/conftest.py
@@ -41,7 +41,8 @@ def download_nltk_resources():
# Download NLTK resources before tests start
download_nltk_resources()
-# Set environment variable to skip external service tests
-os.environ["SKIP_EXTERNAL_SERVICES"] = "true"
+# Default local tests away from external services while allowing selected
+# integration runs to opt in explicitly.
+os.environ.setdefault("SKIP_EXTERNAL_SERVICES", "true")
# Log current Python path for debugging
logging.debug("Python path: %s", sys.path)
diff --git a/hugegraph-llm/src/tests/document/test_document.py b/hugegraph-llm/src/tests/document/test_document.py
index cf106ead..cd5fe5da 100644
--- a/hugegraph-llm/src/tests/document/test_document.py
+++ b/hugegraph-llm/src/tests/document/test_document.py
@@ -17,8 +17,12 @@
import unittest
+import pytest
+
from hugegraph_llm.document import Document, Metadata
+pytestmark = pytest.mark.unit
+
class TestDocument(unittest.TestCase):
def test_document_initialization(self):
diff --git a/hugegraph-llm/src/tests/document/test_document_splitter.py b/hugegraph-llm/src/tests/document/test_document_splitter.py
index d1f67580..e919bddb 100644
--- a/hugegraph-llm/src/tests/document/test_document_splitter.py
+++ b/hugegraph-llm/src/tests/document/test_document_splitter.py
@@ -17,8 +17,12 @@
import unittest
+import pytest
+
from hugegraph_llm.document.chunk_split import ChunkSplitter
+pytestmark = pytest.mark.unit
+
class TestChunkSplitter(unittest.TestCase):
def test_paragraph_split_zh(self):
diff --git a/hugegraph-llm/src/tests/document/test_text_loader.py b/hugegraph-llm/src/tests/document/test_text_loader.py
index f9ac6730..529cfede 100644
--- a/hugegraph-llm/src/tests/document/test_text_loader.py
+++ b/hugegraph-llm/src/tests/document/test_text_loader.py
@@ -19,6 +19,10 @@ import os
import tempfile
import unittest
+import pytest
+
+pytestmark = pytest.mark.unit
+
class TextLoader:
"""Simple text file loader for testing."""
diff --git a/hugegraph-llm/src/tests/middleware/test_middleware.py b/hugegraph-llm/src/tests/middleware/test_middleware.py
index 90e71272..0a1cd36f 100644
--- a/hugegraph-llm/src/tests/middleware/test_middleware.py
+++ b/hugegraph-llm/src/tests/middleware/test_middleware.py
@@ -18,10 +18,13 @@
import unittest
from unittest.mock import AsyncMock, MagicMock, patch
+import pytest
from fastapi import FastAPI
from hugegraph_llm.middleware.middleware import UseTimeMiddleware
+pytestmark = pytest.mark.unit
+
class TestUseTimeMiddlewareInit(unittest.TestCase):
def setUp(self):
diff --git a/hugegraph-llm/src/tests/models/llms/test_litellm_client.py b/hugegraph-llm/src/tests/models/llms/test_litellm_client.py
index 3b27d780..f8ff89f6 100644
--- a/hugegraph-llm/src/tests/models/llms/test_litellm_client.py
+++ b/hugegraph-llm/src/tests/models/llms/test_litellm_client.py
@@ -19,10 +19,13 @@ import asyncio
import unittest
from unittest.mock import AsyncMock, patch
+import pytest
from litellm.exceptions import APIError, BudgetExceededError
from hugegraph_llm.models.llms.litellm import LiteLLMClient
+pytestmark = pytest.mark.contract
+
class TestLiteLLMClient(unittest.TestCase):
def test_budget_exceeded_error_is_not_retried(self):
diff --git a/hugegraph-llm/src/tests/models/llms/test_openai_client.py b/hugegraph-llm/src/tests/models/llms/test_openai_client.py
index 20e5aaac..477e8fb3 100644
--- a/hugegraph-llm/src/tests/models/llms/test_openai_client.py
+++ b/hugegraph-llm/src/tests/models/llms/test_openai_client.py
@@ -19,8 +19,12 @@ import asyncio
import unittest
from unittest.mock import AsyncMock, MagicMock, patch
+import pytest
+
from hugegraph_llm.models.llms.openai import OpenAIClient
+pytestmark = pytest.mark.contract
+
class TestOpenAIClient(unittest.TestCase):
def setUp(self):
diff --git a/hugegraph-llm/src/tests/operators/llm_op/test_disambiguate_data.py b/hugegraph-llm/src/tests/operators/llm_op/test_disambiguate_data.py
index 04ee4214..4bb3c321 100644
--- a/hugegraph-llm/src/tests/operators/llm_op/test_disambiguate_data.py
+++ b/hugegraph-llm/src/tests/operators/llm_op/test_disambiguate_data.py
@@ -17,8 +17,12 @@
import unittest
+import pytest
+
from hugegraph_llm.operators.llm_op.disambiguate_data import DisambiguateData
+pytestmark = pytest.mark.contract
+
class TestDisambiguateData(unittest.TestCase):
def setUp(self):
diff --git a/hugegraph-llm/src/tests/operators/llm_op/test_gremlin_generate.py b/hugegraph-llm/src/tests/operators/llm_op/test_gremlin_generate.py
index 557eade8..fe68428b 100644
--- a/hugegraph-llm/src/tests/operators/llm_op/test_gremlin_generate.py
+++ b/hugegraph-llm/src/tests/operators/llm_op/test_gremlin_generate.py
@@ -21,9 +21,13 @@ import json
import unittest
from unittest.mock import AsyncMock, MagicMock, patch
+import pytest
+
from hugegraph_llm.models.llms.base import BaseLLM
from hugegraph_llm.operators.llm_op.gremlin_generate import GremlinGenerateSynthesize
+pytestmark = pytest.mark.contract
+
class TestGremlinGenerateSynthesize(unittest.TestCase):
@classmethod
diff --git a/hugegraph-llm/src/tests/operators/llm_op/test_info_extract.py b/hugegraph-llm/src/tests/operators/llm_op/test_info_extract.py
index 80359bd5..e352a097 100644
--- a/hugegraph-llm/src/tests/operators/llm_op/test_info_extract.py
+++ b/hugegraph-llm/src/tests/operators/llm_op/test_info_extract.py
@@ -17,12 +17,16 @@
import unittest
+import pytest
+
from hugegraph_llm.operators.llm_op.info_extract import (
InfoExtract,
extract_triples_by_regex,
extract_triples_by_regex_with_schema,
)
+pytestmark = pytest.mark.contract
+
class TestInfoExtract(unittest.TestCase):
def setUp(self):
diff --git a/hugegraph-llm/src/tests/operators/llm_op/test_keyword_extract.py b/hugegraph-llm/src/tests/operators/llm_op/test_keyword_extract.py
index a2cc1bb5..d03c6242 100644
--- a/hugegraph-llm/src/tests/operators/llm_op/test_keyword_extract.py
+++ b/hugegraph-llm/src/tests/operators/llm_op/test_keyword_extract.py
@@ -20,9 +20,13 @@
import unittest
from unittest.mock import MagicMock, patch
+import pytest
+
from hugegraph_llm.models.llms.base import BaseLLM
from hugegraph_llm.operators.llm_op.keyword_extract import KeywordExtract
+pytestmark = pytest.mark.contract
+
class TestKeywordExtract(unittest.TestCase):
def setUp(self):
diff --git a/hugegraph-llm/src/tests/operators/llm_op/test_property_graph_extract.py b/hugegraph-llm/src/tests/operators/llm_op/test_property_graph_extract.py
index 3eb49026..7bef62d5 100644
--- a/hugegraph-llm/src/tests/operators/llm_op/test_property_graph_extract.py
+++ b/hugegraph-llm/src/tests/operators/llm_op/test_property_graph_extract.py
@@ -21,6 +21,8 @@ import json
import unittest
from unittest.mock import MagicMock, patch
+import pytest
+
from hugegraph_llm.models.llms.base import BaseLLM
from hugegraph_llm.operators.llm_op.property_graph_extract import (
PropertyGraphExtract,
@@ -29,6 +31,8 @@ from hugegraph_llm.operators.llm_op.property_graph_extract import (
split_text,
)
+pytestmark = pytest.mark.contract
+
class TestPropertyGraphExtract(unittest.TestCase):
def setUp(self):
diff --git a/hugegraph-python-client/src/tests/api/test_auth.py b/hugegraph-python-client/src/tests/api/test_auth.py
index 1f6dec95..22218ded 100644
--- a/hugegraph-python-client/src/tests/api/test_auth.py
+++ b/hugegraph-python-client/src/tests/api/test_auth.py
@@ -18,10 +18,13 @@
import unittest
+import pytest
from pyhugegraph.utils.exceptions import NotFoundError
from ..client_utils import ClientUtils
+pytestmark = [pytest.mark.integration, pytest.mark.hugegraph]
+
class TestAuthManager(unittest.TestCase):
client = None
diff --git a/hugegraph-python-client/src/tests/api/test_auth_routing.py b/hugegraph-python-client/src/tests/api/test_auth_routing.py
index e7a6e105..1fcd5a3d 100644
--- a/hugegraph-python-client/src/tests/api/test_auth_routing.py
+++ b/hugegraph-python-client/src/tests/api/test_auth_routing.py
@@ -20,6 +20,8 @@ from urllib.parse import urljoin
import pytest
from pyhugegraph.api.auth import AuthManager
+pytestmark = pytest.mark.contract
+
class DummyCfg:
def __init__(self, url, graphspace, gs_supported, graph_name):
diff --git a/hugegraph-python-client/src/tests/api/test_graph.py b/hugegraph-python-client/src/tests/api/test_graph.py
index a66c3fb9..e980cc30 100644
--- a/hugegraph-python-client/src/tests/api/test_graph.py
+++ b/hugegraph-python-client/src/tests/api/test_graph.py
@@ -17,10 +17,13 @@
import unittest
+import pytest
from pyhugegraph.utils.exceptions import NotFoundError
from ..client_utils import ClientUtils
+pytestmark = [pytest.mark.integration, pytest.mark.hugegraph]
+
class TestGraphManager(unittest.TestCase):
client = None
diff --git a/hugegraph-python-client/src/tests/api/test_graphs.py b/hugegraph-python-client/src/tests/api/test_graphs.py
index 13fe53b0..54bf10b6 100644
--- a/hugegraph-python-client/src/tests/api/test_graphs.py
+++ b/hugegraph-python-client/src/tests/api/test_graphs.py
@@ -17,8 +17,12 @@
import unittest
+import pytest
+
from ..client_utils import ClientUtils
+pytestmark = [pytest.mark.integration, pytest.mark.hugegraph]
+
class TestGraphsManager(unittest.TestCase):
client = None
diff --git a/hugegraph-python-client/src/tests/api/test_gremlin.py b/hugegraph-python-client/src/tests/api/test_gremlin.py
index 346812b5..6e53cb3e 100644
--- a/hugegraph-python-client/src/tests/api/test_gremlin.py
+++ b/hugegraph-python-client/src/tests/api/test_gremlin.py
@@ -24,6 +24,8 @@ from pyhugegraph.utils.exceptions import NotFoundError
from ..client_utils import ClientUtils
+pytestmark = [pytest.mark.integration, pytest.mark.hugegraph]
+
class TestGremlin(unittest.TestCase):
client = None
diff --git a/hugegraph-python-client/src/tests/api/test_metric.py b/hugegraph-python-client/src/tests/api/test_metric.py
index 3e1a67ad..4970152d 100644
--- a/hugegraph-python-client/src/tests/api/test_metric.py
+++ b/hugegraph-python-client/src/tests/api/test_metric.py
@@ -17,8 +17,12 @@
import unittest
+import pytest
+
from ..client_utils import ClientUtils
+pytestmark = [pytest.mark.integration, pytest.mark.hugegraph]
+
class TestMetricsManager(unittest.TestCase):
client = None
diff --git a/hugegraph-python-client/src/tests/api/test_response_validation.py b/hugegraph-python-client/src/tests/api/test_response_validation.py
index 759d9718..da8e771c 100644
--- a/hugegraph-python-client/src/tests/api/test_response_validation.py
+++ b/hugegraph-python-client/src/tests/api/test_response_validation.py
@@ -18,9 +18,12 @@
import unittest
from unittest.mock import Mock
+import pytest
import requests
from pyhugegraph.utils.util import ResponseValidation
+pytestmark = pytest.mark.contract
+
class TestResponseValidation(unittest.TestCase):
def _mock_error_response(self, body, text):
diff --git a/hugegraph-python-client/src/tests/api/test_schema.py b/hugegraph-python-client/src/tests/api/test_schema.py
index 4f91822c..4dae853b 100644
--- a/hugegraph-python-client/src/tests/api/test_schema.py
+++ b/hugegraph-python-client/src/tests/api/test_schema.py
@@ -17,8 +17,12 @@
import unittest
+import pytest
+
from ..client_utils import ClientUtils
+pytestmark = [pytest.mark.integration, pytest.mark.hugegraph]
+
class TestSchemaManager(unittest.TestCase):
client = None
diff --git a/hugegraph-python-client/src/tests/api/test_task.py b/hugegraph-python-client/src/tests/api/test_task.py
index 599d8d1f..f5d3a2a4 100644
--- a/hugegraph-python-client/src/tests/api/test_task.py
+++ b/hugegraph-python-client/src/tests/api/test_task.py
@@ -17,10 +17,13 @@
import unittest
+import pytest
from pyhugegraph.utils.exceptions import NotFoundError
from ..client_utils import ClientUtils
+pytestmark = [pytest.mark.integration, pytest.mark.hugegraph]
+
class TestTaskManager(unittest.TestCase):
client = None
diff --git a/hugegraph-python-client/src/tests/api/test_traverser.py b/hugegraph-python-client/src/tests/api/test_traverser.py
index bcd40acf..03106966 100644
--- a/hugegraph-python-client/src/tests/api/test_traverser.py
+++ b/hugegraph-python-client/src/tests/api/test_traverser.py
@@ -17,8 +17,12 @@
import unittest
+import pytest
+
from ..client_utils import ClientUtils
+pytestmark = [pytest.mark.integration, pytest.mark.hugegraph]
+
class TestTraverserManager(unittest.TestCase):
client = None
diff --git a/hugegraph-python-client/src/tests/api/test_variable.py b/hugegraph-python-client/src/tests/api/test_variable.py
index d75ac514..cdb86022 100644
--- a/hugegraph-python-client/src/tests/api/test_variable.py
+++ b/hugegraph-python-client/src/tests/api/test_variable.py
@@ -22,6 +22,8 @@ from pyhugegraph.utils.exceptions import NotFoundError
from ..client_utils import ClientUtils
+pytestmark = [pytest.mark.integration, pytest.mark.hugegraph]
+
class TestVariable(unittest.TestCase):
client = None
diff --git a/hugegraph-python-client/src/tests/api/test_version.py b/hugegraph-python-client/src/tests/api/test_version.py
index 1ca4a1e2..71eb0562 100644
--- a/hugegraph-python-client/src/tests/api/test_version.py
+++ b/hugegraph-python-client/src/tests/api/test_version.py
@@ -17,8 +17,12 @@
import unittest
+import pytest
+
from ..client_utils import ClientUtils
+pytestmark = [pytest.mark.integration, pytest.mark.hugegraph]
+
class TestVersion(unittest.TestCase):
client = None
diff --git a/pyproject.toml b/pyproject.toml
index faa8c73c..2687ce93 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -144,6 +144,18 @@ constraint-dependencies = [
"python-dateutil~=2.9.0",
]
+[tool.pytest.ini_options]
+markers = [
+ "unit: fast deterministic tests without network or Docker",
+ "contract: public contract tests; may use mocks but verify stable behavior",
+ "integration: tests requiring a real local service such as HugeGraph",
+ "hugegraph: tests requiring HugeGraph Server",
+ "smoke: end-to-end-ish high-value smoke over production pipeline boundaries",
+ "external: tests requiring external provider credentials or non-HugeGraph services",
+ "slow: long-running tests excluded from default local loops",
+]
+addopts = "--strict-markers --strict-config"
+
# for code format
[tool.ruff]
line-length = 120