You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ak...@apache.org on 2026/06/02 19:48:00 UTC
(tooling-gofannon) branch main updated: add bedrock config and adaptive-thinking routing for opus 4.7/4.8
This is an automated email from the ASF dual-hosted git repository.
andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-gofannon.git
The following commit(s) were added to refs/heads/main by this push:
new 3fcfca5 add bedrock config and adaptive-thinking routing for opus 4.7/4.8
3fcfca5 is described below
commit 3fcfca57cd5ead236f130897fc756fdd12d6b374
Author: Andrew Musselman <ak...@apache.org>
AuthorDate: Tue Jun 2 12:47:46 2026 -0700
add bedrock config and adaptive-thinking routing for opus 4.7/4.8
Register the two models with supports_thinking/supports_effort/
returns_thoughts and extended reasoning_effort choices (xhigh, max).
In llm_service.py, branch on _is_opus_4_7_or_later and strip
reasoning_effort, bypassing litellm's legacy thinking.type.enabled
translation that Bedrock rejects for these models. Older models and
non-Opus Claudes still pass reasoning_effort through unchanged.
Verified end-to-end against Bedrock. Tests cover the helper, both
kwargs branches, and the disable/absent cases.
---
.../api/user-service/config/bedrock/__init__.py | 85 ++++++++++++
.../api/user-service/services/llm_service.py | 20 ++-
.../tests/unit/services/test_llm_service.py | 145 +++++++++++++++++++++
.../tests/unit/test_provider_config.py | 35 +++++
4 files changed, 284 insertions(+), 1 deletion(-)
diff --git a/webapp/packages/api/user-service/config/bedrock/__init__.py b/webapp/packages/api/user-service/config/bedrock/__init__.py
index 54bad64..2777477 100644
--- a/webapp/packages/api/user-service/config/bedrock/__init__.py
+++ b/webapp/packages/api/user-service/config/bedrock/__init__.py
@@ -7,6 +7,91 @@
# Note: Claude 4.5 models cannot use both temperature and top_p together
models = {
+ # =========================================================================
+ # Anthropic Claude Opus 4.8 (Bedrock)
+ # Uses the new adaptive thinking format introduced with Opus 4.7:
+ # thinking={"type":"adaptive"}, output_config={"effort":"high"}
+ # NOT the legacy thinking={"type":"enabled","budget_tokens":N} format.
+ # llm_service.py must branch on the model string to send the right shape.
+ # =========================================================================
+ "us.anthropic.claude-opus-4-8": {
+ "returns_thoughts": True,
+ "supports_effort": True,
+ "supports_thinking": True,
+ "context_window": 200000,
+ "parameters": {
+ "temperature": {
+ "type": "float",
+ "default": 1.0,
+ "min": 0.0,
+ "max": 1.0,
+ "description": "Randomness (0=focused, 1=creative). Locked to 1.0 when thinking enabled.",
+ "mutually_exclusive_with": ["top_p"]
+ },
+ "top_p": {
+ "type": "float",
+ "default": 0.9,
+ "min": 0.0,
+ "max": 1.0,
+ "description": "Nucleus sampling (0.1=conservative, 0.95=diverse)",
+ "mutually_exclusive_with": ["temperature"]
+ },
+ "reasoning_effort": {
+ "type": "choice",
+ "default": "disable",
+ "choices": ["disable", "low", "medium", "high", "xhigh", "max"],
+ "description": "Reasoning Effort: Enables extended thinking. xhigh/max are Opus 4.7+ only."
+ },
+ "max_tokens": {
+ "type": "integer",
+ "default": 16384,
+ "min": 1,
+ "max": 128000,
+ "description": "Maximum tokens in response"
+ },
+ }
+ },
+ # =========================================================================
+ # Anthropic Claude Opus 4.7 (Bedrock)
+ # Same adaptive thinking format as 4.8 — see comment block above.
+ # =========================================================================
+ "us.anthropic.claude-opus-4-7": {
+ "returns_thoughts": True,
+ "supports_effort": True,
+ "supports_thinking": True,
+ "context_window": 200000,
+ "parameters": {
+ "temperature": {
+ "type": "float",
+ "default": 1.0,
+ "min": 0.0,
+ "max": 1.0,
+ "description": "Randomness (0=focused, 1=creative). Locked to 1.0 when thinking enabled.",
+ "mutually_exclusive_with": ["top_p"]
+ },
+ "top_p": {
+ "type": "float",
+ "default": 0.9,
+ "min": 0.0,
+ "max": 1.0,
+ "description": "Nucleus sampling (0.1=conservative, 0.95=diverse)",
+ "mutually_exclusive_with": ["temperature"]
+ },
+ "reasoning_effort": {
+ "type": "choice",
+ "default": "disable",
+ "choices": ["disable", "low", "medium", "high", "xhigh", "max"],
+ "description": "Reasoning Effort: Enables extended thinking. xhigh/max are Opus 4.7+ only."
+ },
+ "max_tokens": {
+ "type": "integer",
+ "default": 16384,
+ "min": 1,
+ "max": 128000,
+ "description": "Maximum tokens in response"
+ },
+ }
+ },
# =========================================================================
# Anthropic Claude Opus 4.6 (Bedrock)
# =========================================================================
diff --git a/webapp/packages/api/user-service/services/llm_service.py b/webapp/packages/api/user-service/services/llm_service.py
index bcf7af2..15f04c6 100644
--- a/webapp/packages/api/user-service/services/llm_service.py
+++ b/webapp/packages/api/user-service/services/llm_service.py
@@ -56,6 +56,16 @@ def _extract_response_cost(response_obj: Any) -> Optional[float]:
return None
return None
+def _is_opus_4_7_or_later(model: str) -> bool:
+ """Opus 4.7+ uses adaptive thinking with output_config.effort
+ instead of the old reasoning_effort -> thinking.type.enabled
+ translation that litellm does for Opus 4.6 and earlier."""
+ if "claude-opus" not in model:
+ return False
+ if "claude-opus-4-6" in model:
+ return False
+ return any(tag in model for tag in ("claude-opus-4-7", "claude-opus-4-8"))
+
async def call_llm(
provider: str,
@@ -305,7 +315,15 @@ async def call_llm(
else:
# Standard acompletion call for most models
if reasoning_effort != 'disable':
- kwargs['reasoning_effort'] = reasoning_effort
+ if _is_opus_4_7_or_later(model):
+ # Opus 4.7+ rejects the old thinking.type.enabled format
+ # that litellm's reasoning_effort translation produces.
+ # Set the adaptive-thinking params directly and skip
+ # litellm's translation.
+ kwargs['thinking'] = {'type': 'adaptive'}
+ kwargs['output_config'] = {'effort': reasoning_effort}
+ else:
+ kwargs['reasoning_effort'] = reasoning_effort
# Retry loop. Two exception classes are retryable here:
# - litellm.Timeout — slow network or model; modest linear backoff.
diff --git a/webapp/packages/api/user-service/tests/unit/services/test_llm_service.py b/webapp/packages/api/user-service/tests/unit/services/test_llm_service.py
index bf84a4a..67f0207 100644
--- a/webapp/packages/api/user-service/tests/unit/services/test_llm_service.py
+++ b/webapp/packages/api/user-service/tests/unit/services/test_llm_service.py
@@ -534,3 +534,148 @@ class TestBedrockProvider:
assert call_kwargs["temperature"] == 0.7
assert call_kwargs["max_tokens"] == 1000
+
+class TestIsOpus47OrLater:
+ """Tests for the _is_opus_4_7_or_later helper."""
+
+ @pytest.mark.parametrize("model", [
+ "us.anthropic.claude-opus-4-7",
+ "us.anthropic.claude-opus-4-8",
+ # Tolerate either bare or prefixed model strings.
+ "bedrock/us.anthropic.claude-opus-4-7",
+ "bedrock/us.anthropic.claude-opus-4-8",
+ ])
+ def test_returns_true_for_opus_4_7_and_4_8(self, model):
+ assert llm_service._is_opus_4_7_or_later(model) is True
+
+ @pytest.mark.parametrize("model", [
+ # Older Opus generations use the legacy thinking format.
+ "us.anthropic.claude-opus-4-6-v1",
+ "us.anthropic.claude-opus-4-5-20251101-v1:0",
+ "us.anthropic.claude-opus-4-1-20250805-v1:0",
+ # Non-Opus Claude models never go through the new format,
+ # regardless of generation.
+ "us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+ "us.anthropic.claude-haiku-4-5-20251001-v1:0",
+ # Completely unrelated providers / models.
+ "gpt-5",
+ "",
+ ])
+ def test_returns_false_for_older_opus_and_non_opus(self, model):
+ assert llm_service._is_opus_4_7_or_later(model) is False
+
+
+class TestCallLlmAdaptiveThinkingKwargs:
+ """Verify the kwargs that reach litellm.acompletion when reasoning_effort
+ is set on Opus 4.7+ vs. older models. Locks in the parameter-shape
+ contract that Bedrock's converse API enforces."""
+
+ @pytest.fixture
+ def captured(self):
+ """Holds the kwargs the fake acompletion saw, for assertion."""
+ return {}
+
+ @pytest.fixture
+ def patched_llm(self, monkeypatch, captured):
+ async def capturing_acompletion(**kwargs):
+ captured.update(kwargs)
+ return _DummyResponse("ok", total_cost=0.0)
+ monkeypatch.setattr(llm_service.litellm, "acompletion", capturing_acompletion)
+ monkeypatch.setattr(llm_service, "get_observability_service", lambda: Mock())
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("model,effort", [
+ ("us.anthropic.claude-opus-4-7", "low"),
+ ("us.anthropic.claude-opus-4-7", "medium"),
+ ("us.anthropic.claude-opus-4-7", "high"),
+ ("us.anthropic.claude-opus-4-7", "xhigh"),
+ ("us.anthropic.claude-opus-4-7", "max"),
+ ("us.anthropic.claude-opus-4-8", "high"),
+ ])
+ async def test_opus_4_7_plus_with_effort_uses_adaptive_thinking(
+ self, patched_llm, captured, model, effort,
+ ):
+ """For Opus 4.7+, reasoning_effort must be translated into
+ thinking={"type":"adaptive"} and output_config={"effort":...}
+ before litellm sees the call. The legacy `reasoning_effort`
+ kwarg must NOT be passed through, otherwise litellm's internal
+ translation re-injects the rejected thinking.type.enabled shape."""
+ await llm_service.call_llm(
+ provider="bedrock",
+ model=model,
+ messages=[{"role": "user", "content": "hi"}],
+ parameters={"reasoning_effort": effort},
+ user_service=Mock(),
+ user_id="user-1",
+ )
+ assert captured.get("thinking") == {"type": "adaptive"}
+ assert captured.get("output_config") == {"effort": effort}
+ assert "reasoning_effort" not in captured, (
+ "reasoning_effort must be stripped on 4.7+; if it leaks "
+ "through, litellm re-translates it to thinking.type.enabled "
+ "and Bedrock 400s."
+ )
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("model", [
+ "us.anthropic.claude-opus-4-6-v1",
+ "us.anthropic.claude-opus-4-5-20251101-v1:0",
+ "us.anthropic.claude-opus-4-1-20250805-v1:0",
+ "us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+ "us.anthropic.claude-haiku-4-5-20251001-v1:0",
+ ])
+ async def test_older_models_pass_reasoning_effort_through(
+ self, patched_llm, captured, model,
+ ):
+ """4.6 and earlier (and all non-Opus Claudes) keep the legacy
+ path: reasoning_effort passes through to litellm unchanged, and
+ no adaptive-thinking params are set."""
+ await llm_service.call_llm(
+ provider="bedrock",
+ model=model,
+ messages=[{"role": "user", "content": "hi"}],
+ parameters={"reasoning_effort": "high"},
+ user_service=Mock(),
+ user_id="user-1",
+ )
+ assert captured.get("reasoning_effort") == "high"
+ assert "thinking" not in captured
+ assert "output_config" not in captured
+
+ @pytest.mark.asyncio
+ async def test_opus_4_8_with_disable_sets_no_thinking_params(
+ self, patched_llm, captured,
+ ):
+ """reasoning_effort='disable' must skip BOTH branches — no
+ thinking dict, no output_config, no reasoning_effort kwarg.
+ Otherwise the model thinks when the caller asked it not to."""
+ await llm_service.call_llm(
+ provider="bedrock",
+ model="us.anthropic.claude-opus-4-8",
+ messages=[{"role": "user", "content": "hi"}],
+ parameters={"reasoning_effort": "disable"},
+ user_service=Mock(),
+ user_id="user-1",
+ )
+ assert "thinking" not in captured
+ assert "output_config" not in captured
+ assert "reasoning_effort" not in captured
+
+ @pytest.mark.asyncio
+ async def test_opus_4_8_without_reasoning_effort_parameter(
+ self, patched_llm, captured,
+ ):
+ """When reasoning_effort isn't supplied at all (no key in the
+ parameters dict), behavior should match 'disable' — no thinking
+ params injected."""
+ await llm_service.call_llm(
+ provider="bedrock",
+ model="us.anthropic.claude-opus-4-8",
+ messages=[{"role": "user", "content": "hi"}],
+ parameters={},
+ user_service=Mock(),
+ user_id="user-1",
+ )
+ assert "thinking" not in captured
+ assert "output_config" not in captured
+ assert "reasoning_effort" not in captured
\ No newline at end of file
diff --git a/webapp/packages/api/user-service/tests/unit/test_provider_config.py b/webapp/packages/api/user-service/tests/unit/test_provider_config.py
index e6f4619..0e584f5 100644
--- a/webapp/packages/api/user-service/tests/unit/test_provider_config.py
+++ b/webapp/packages/api/user-service/tests/unit/test_provider_config.py
@@ -81,6 +81,41 @@ class TestBedrockModels:
assert "reasoning_effort" in params
assert params["reasoning_effort"]["choices"] == ["disable", "low", "medium", "high"]
+ @pytest.mark.parametrize("model_id", [
+ "us.anthropic.claude-opus-4-7",
+ "us.anthropic.claude-opus-4-8",
+ ])
+ def test_opus_4_7_plus_models_registered(self, bedrock_models, model_id):
+ """Opus 4.7+ models are present with the metadata flags the LLM
+ service layer needs to route through the new adaptive thinking
+ parameter format."""
+ assert model_id in bedrock_models
+ entry = bedrock_models[model_id]
+ assert entry["supports_thinking"] is True
+ assert entry["supports_effort"] is True
+ assert entry["returns_thoughts"] is True
+ assert entry["context_window"] >= 200000
+
+ @pytest.mark.parametrize("model_id", [
+ "us.anthropic.claude-opus-4-7",
+ "us.anthropic.claude-opus-4-8",
+ ])
+ def test_opus_4_7_plus_models_expose_extended_effort_levels(self, bedrock_models, model_id):
+ """Opus 4.7 introduced two new effort levels (xhigh, max) on top of
+ the legacy low/medium/high. The config must surface them so the UI
+ picker exposes them; absence means the model can't be invoked at
+ its higher reasoning tiers from the SPA."""
+ params = bedrock_models[model_id]["parameters"]
+ assert "reasoning_effort" in params
+ choices = params["reasoning_effort"]["choices"]
+ # New tiers are present
+ assert "xhigh" in choices, f"{model_id} missing xhigh effort"
+ assert "max" in choices, f"{model_id} missing max effort"
+ # Legacy tiers still available so existing prompts that say
+ # reasoning_effort="high" don't break.
+ for legacy in ("disable", "low", "medium", "high"):
+ assert legacy in choices, f"{model_id} dropped legacy {legacy!r}"
+
@pytest.mark.parametrize("model_id", [
# Claude 3.5 and 3 don't support extended thinking
"us.anthropic.claude-3-5-haiku-20241022-v1:0",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org