You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ak...@apache.org on 2026/06/02 19:48:00 UTC

(tooling-gofannon) branch main updated: add bedrock config and adaptive-thinking routing for opus 4.7/4.8

This is an automated email from the ASF dual-hosted git repository.

andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-gofannon.git


The following commit(s) were added to refs/heads/main by this push:
     new 3fcfca5  add bedrock config and adaptive-thinking routing for opus 4.7/4.8
3fcfca5 is described below

commit 3fcfca57cd5ead236f130897fc756fdd12d6b374
Author: Andrew Musselman <ak...@apache.org>
AuthorDate: Tue Jun 2 12:47:46 2026 -0700

    add bedrock config and adaptive-thinking routing for opus 4.7/4.8
    
    Register the two models with supports_thinking/supports_effort/
    returns_thoughts and extended reasoning_effort choices (xhigh, max).
    In llm_service.py, branch on _is_opus_4_7_or_later and strip
    reasoning_effort, bypassing litellm's legacy thinking.type.enabled
    translation that Bedrock rejects for these models. Older models and
    non-Opus Claudes still pass reasoning_effort through unchanged.
    Verified end-to-end against Bedrock. Tests cover the helper, both
    kwargs branches, and the disable/absent cases.
---
 .../api/user-service/config/bedrock/__init__.py    |  85 ++++++++++++
 .../api/user-service/services/llm_service.py       |  20 ++-
 .../tests/unit/services/test_llm_service.py        | 145 +++++++++++++++++++++
 .../tests/unit/test_provider_config.py             |  35 +++++
 4 files changed, 284 insertions(+), 1 deletion(-)

diff --git a/webapp/packages/api/user-service/config/bedrock/__init__.py b/webapp/packages/api/user-service/config/bedrock/__init__.py
index 54bad64..2777477 100644
--- a/webapp/packages/api/user-service/config/bedrock/__init__.py
+++ b/webapp/packages/api/user-service/config/bedrock/__init__.py
@@ -7,6 +7,91 @@
 # Note: Claude 4.5 models cannot use both temperature and top_p together
 
 models = {
+    # =========================================================================
+    # Anthropic Claude Opus 4.8 (Bedrock)
+    # Uses the new adaptive thinking format introduced with Opus 4.7:
+    #   thinking={"type":"adaptive"}, output_config={"effort":"high"}
+    # NOT the legacy thinking={"type":"enabled","budget_tokens":N} format.
+    # llm_service.py must branch on the model string to send the right shape.
+    # =========================================================================
+    "us.anthropic.claude-opus-4-8": {
+        "returns_thoughts": True,
+        "supports_effort": True,
+        "supports_thinking": True,
+        "context_window": 200000,
+        "parameters": {
+            "temperature": {
+                "type": "float",
+                "default": 1.0,
+                "min": 0.0,
+                "max": 1.0,
+                "description": "Randomness (0=focused, 1=creative). Locked to 1.0 when thinking enabled.",
+                "mutually_exclusive_with": ["top_p"]
+            },
+            "top_p": {
+                "type": "float",
+                "default": 0.9,
+                "min": 0.0,
+                "max": 1.0,
+                "description": "Nucleus sampling (0.1=conservative, 0.95=diverse)",
+                "mutually_exclusive_with": ["temperature"]
+            },
+            "reasoning_effort": {
+                "type": "choice",
+                "default": "disable",
+                "choices": ["disable", "low", "medium", "high", "xhigh", "max"],
+                "description": "Reasoning Effort: Enables extended thinking. xhigh/max are Opus 4.7+ only."
+            },
+            "max_tokens": {
+                "type": "integer",
+                "default": 16384,
+                "min": 1,
+                "max": 128000,
+                "description": "Maximum tokens in response"
+            },
+        }
+    },
+    # =========================================================================
+    # Anthropic Claude Opus 4.7 (Bedrock)
+    # Same adaptive thinking format as 4.8 — see comment block above.
+    # =========================================================================
+    "us.anthropic.claude-opus-4-7": {
+        "returns_thoughts": True,
+        "supports_effort": True,
+        "supports_thinking": True,
+        "context_window": 200000,
+        "parameters": {
+            "temperature": {
+                "type": "float",
+                "default": 1.0,
+                "min": 0.0,
+                "max": 1.0,
+                "description": "Randomness (0=focused, 1=creative). Locked to 1.0 when thinking enabled.",
+                "mutually_exclusive_with": ["top_p"]
+            },
+            "top_p": {
+                "type": "float",
+                "default": 0.9,
+                "min": 0.0,
+                "max": 1.0,
+                "description": "Nucleus sampling (0.1=conservative, 0.95=diverse)",
+                "mutually_exclusive_with": ["temperature"]
+            },
+            "reasoning_effort": {
+                "type": "choice",
+                "default": "disable",
+                "choices": ["disable", "low", "medium", "high", "xhigh", "max"],
+                "description": "Reasoning Effort: Enables extended thinking. xhigh/max are Opus 4.7+ only."
+            },
+            "max_tokens": {
+                "type": "integer",
+                "default": 16384,
+                "min": 1,
+                "max": 128000,
+                "description": "Maximum tokens in response"
+            },
+        }
+    },
     # =========================================================================
     # Anthropic Claude Opus 4.6 (Bedrock)
     # =========================================================================
diff --git a/webapp/packages/api/user-service/services/llm_service.py b/webapp/packages/api/user-service/services/llm_service.py
index bcf7af2..15f04c6 100644
--- a/webapp/packages/api/user-service/services/llm_service.py
+++ b/webapp/packages/api/user-service/services/llm_service.py
@@ -56,6 +56,16 @@ def _extract_response_cost(response_obj: Any) -> Optional[float]:
             return None
     return None
 
+def _is_opus_4_7_or_later(model: str) -> bool:
+    """Opus 4.7+ uses adaptive thinking with output_config.effort
+    instead of the old reasoning_effort -> thinking.type.enabled
+    translation that litellm does for Opus 4.6 and earlier."""
+    if "claude-opus" not in model:
+        return False
+    if "claude-opus-4-6" in model:
+        return False
+    return any(tag in model for tag in ("claude-opus-4-7", "claude-opus-4-8"))
+
 
 async def call_llm(
     provider: str,
@@ -305,7 +315,15 @@ async def call_llm(
     else:
         # Standard acompletion call for most models
         if reasoning_effort != 'disable':
-            kwargs['reasoning_effort'] = reasoning_effort
+            if _is_opus_4_7_or_later(model):
+                # Opus 4.7+ rejects the old thinking.type.enabled format
+                # that litellm's reasoning_effort translation produces.
+                # Set the adaptive-thinking params directly and skip
+                # litellm's translation.
+                kwargs['thinking'] = {'type': 'adaptive'}
+                kwargs['output_config'] = {'effort': reasoning_effort}
+            else:
+                kwargs['reasoning_effort'] = reasoning_effort
 
         # Retry loop. Two exception classes are retryable here:
         #   - litellm.Timeout       — slow network or model; modest linear backoff.
diff --git a/webapp/packages/api/user-service/tests/unit/services/test_llm_service.py b/webapp/packages/api/user-service/tests/unit/services/test_llm_service.py
index bf84a4a..67f0207 100644
--- a/webapp/packages/api/user-service/tests/unit/services/test_llm_service.py
+++ b/webapp/packages/api/user-service/tests/unit/services/test_llm_service.py
@@ -534,3 +534,148 @@ class TestBedrockProvider:
 
         assert call_kwargs["temperature"] == 0.7
         assert call_kwargs["max_tokens"] == 1000
+
+class TestIsOpus47OrLater:
+    """Tests for the _is_opus_4_7_or_later helper."""
+
+    @pytest.mark.parametrize("model", [
+        "us.anthropic.claude-opus-4-7",
+        "us.anthropic.claude-opus-4-8",
+        # Tolerate either bare or prefixed model strings.
+        "bedrock/us.anthropic.claude-opus-4-7",
+        "bedrock/us.anthropic.claude-opus-4-8",
+    ])
+    def test_returns_true_for_opus_4_7_and_4_8(self, model):
+        assert llm_service._is_opus_4_7_or_later(model) is True
+
+    @pytest.mark.parametrize("model", [
+        # Older Opus generations use the legacy thinking format.
+        "us.anthropic.claude-opus-4-6-v1",
+        "us.anthropic.claude-opus-4-5-20251101-v1:0",
+        "us.anthropic.claude-opus-4-1-20250805-v1:0",
+        # Non-Opus Claude models never go through the new format,
+        # regardless of generation.
+        "us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+        "us.anthropic.claude-haiku-4-5-20251001-v1:0",
+        # Completely unrelated providers / models.
+        "gpt-5",
+        "",
+    ])
+    def test_returns_false_for_older_opus_and_non_opus(self, model):
+        assert llm_service._is_opus_4_7_or_later(model) is False
+
+
+class TestCallLlmAdaptiveThinkingKwargs:
+    """Verify the kwargs that reach litellm.acompletion when reasoning_effort
+    is set on Opus 4.7+ vs. older models. Locks in the parameter-shape
+    contract that Bedrock's converse API enforces."""
+
+    @pytest.fixture
+    def captured(self):
+        """Holds the kwargs the fake acompletion saw, for assertion."""
+        return {}
+
+    @pytest.fixture
+    def patched_llm(self, monkeypatch, captured):
+        async def capturing_acompletion(**kwargs):
+            captured.update(kwargs)
+            return _DummyResponse("ok", total_cost=0.0)
+        monkeypatch.setattr(llm_service.litellm, "acompletion", capturing_acompletion)
+        monkeypatch.setattr(llm_service, "get_observability_service", lambda: Mock())
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("model,effort", [
+        ("us.anthropic.claude-opus-4-7", "low"),
+        ("us.anthropic.claude-opus-4-7", "medium"),
+        ("us.anthropic.claude-opus-4-7", "high"),
+        ("us.anthropic.claude-opus-4-7", "xhigh"),
+        ("us.anthropic.claude-opus-4-7", "max"),
+        ("us.anthropic.claude-opus-4-8", "high"),
+    ])
+    async def test_opus_4_7_plus_with_effort_uses_adaptive_thinking(
+        self, patched_llm, captured, model, effort,
+    ):
+        """For Opus 4.7+, reasoning_effort must be translated into
+        thinking={"type":"adaptive"} and output_config={"effort":...}
+        before litellm sees the call. The legacy `reasoning_effort`
+        kwarg must NOT be passed through, otherwise litellm's internal
+        translation re-injects the rejected thinking.type.enabled shape."""
+        await llm_service.call_llm(
+            provider="bedrock",
+            model=model,
+            messages=[{"role": "user", "content": "hi"}],
+            parameters={"reasoning_effort": effort},
+            user_service=Mock(),
+            user_id="user-1",
+        )
+        assert captured.get("thinking") == {"type": "adaptive"}
+        assert captured.get("output_config") == {"effort": effort}
+        assert "reasoning_effort" not in captured, (
+            "reasoning_effort must be stripped on 4.7+; if it leaks "
+            "through, litellm re-translates it to thinking.type.enabled "
+            "and Bedrock 400s."
+        )
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("model", [
+        "us.anthropic.claude-opus-4-6-v1",
+        "us.anthropic.claude-opus-4-5-20251101-v1:0",
+        "us.anthropic.claude-opus-4-1-20250805-v1:0",
+        "us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+        "us.anthropic.claude-haiku-4-5-20251001-v1:0",
+    ])
+    async def test_older_models_pass_reasoning_effort_through(
+        self, patched_llm, captured, model,
+    ):
+        """4.6 and earlier (and all non-Opus Claudes) keep the legacy
+        path: reasoning_effort passes through to litellm unchanged, and
+        no adaptive-thinking params are set."""
+        await llm_service.call_llm(
+            provider="bedrock",
+            model=model,
+            messages=[{"role": "user", "content": "hi"}],
+            parameters={"reasoning_effort": "high"},
+            user_service=Mock(),
+            user_id="user-1",
+        )
+        assert captured.get("reasoning_effort") == "high"
+        assert "thinking" not in captured
+        assert "output_config" not in captured
+
+    @pytest.mark.asyncio
+    async def test_opus_4_8_with_disable_sets_no_thinking_params(
+        self, patched_llm, captured,
+    ):
+        """reasoning_effort='disable' must skip BOTH branches — no
+        thinking dict, no output_config, no reasoning_effort kwarg.
+        Otherwise the model thinks when the caller asked it not to."""
+        await llm_service.call_llm(
+            provider="bedrock",
+            model="us.anthropic.claude-opus-4-8",
+            messages=[{"role": "user", "content": "hi"}],
+            parameters={"reasoning_effort": "disable"},
+            user_service=Mock(),
+            user_id="user-1",
+        )
+        assert "thinking" not in captured
+        assert "output_config" not in captured
+        assert "reasoning_effort" not in captured
+
+    @pytest.mark.asyncio
+    async def test_opus_4_8_without_reasoning_effort_parameter(
+        self, patched_llm, captured,
+    ):
+        """When reasoning_effort isn't supplied at all (no key in the
+        parameters dict), behavior should match 'disable' — no thinking
+        params injected."""
+        await llm_service.call_llm(
+            provider="bedrock",
+            model="us.anthropic.claude-opus-4-8",
+            messages=[{"role": "user", "content": "hi"}],
+            parameters={},
+            user_service=Mock(),
+            user_id="user-1",
+        )
+        assert "thinking" not in captured
+        assert "output_config" not in captured
+        assert "reasoning_effort" not in captured
\ No newline at end of file
diff --git a/webapp/packages/api/user-service/tests/unit/test_provider_config.py b/webapp/packages/api/user-service/tests/unit/test_provider_config.py
index e6f4619..0e584f5 100644
--- a/webapp/packages/api/user-service/tests/unit/test_provider_config.py
+++ b/webapp/packages/api/user-service/tests/unit/test_provider_config.py
@@ -81,6 +81,41 @@ class TestBedrockModels:
         assert "reasoning_effort" in params
         assert params["reasoning_effort"]["choices"] == ["disable", "low", "medium", "high"]
 
+    @pytest.mark.parametrize("model_id", [
+        "us.anthropic.claude-opus-4-7",
+        "us.anthropic.claude-opus-4-8",
+    ])
+    def test_opus_4_7_plus_models_registered(self, bedrock_models, model_id):
+        """Opus 4.7+ models are present with the metadata flags the LLM
+        service layer needs to route through the new adaptive thinking
+        parameter format."""
+        assert model_id in bedrock_models
+        entry = bedrock_models[model_id]
+        assert entry["supports_thinking"] is True
+        assert entry["supports_effort"] is True
+        assert entry["returns_thoughts"] is True
+        assert entry["context_window"] >= 200000
+
+    @pytest.mark.parametrize("model_id", [
+        "us.anthropic.claude-opus-4-7",
+        "us.anthropic.claude-opus-4-8",
+    ])
+    def test_opus_4_7_plus_models_expose_extended_effort_levels(self, bedrock_models, model_id):
+        """Opus 4.7 introduced two new effort levels (xhigh, max) on top of
+        the legacy low/medium/high. The config must surface them so the UI
+        picker exposes them; absence means the model can't be invoked at
+        its higher reasoning tiers from the SPA."""
+        params = bedrock_models[model_id]["parameters"]
+        assert "reasoning_effort" in params
+        choices = params["reasoning_effort"]["choices"]
+        # New tiers are present
+        assert "xhigh" in choices, f"{model_id} missing xhigh effort"
+        assert "max" in choices, f"{model_id} missing max effort"
+        # Legacy tiers still available so existing prompts that say
+        # reasoning_effort="high" don't break.
+        for legacy in ("disable", "low", "medium", "high"):
+            assert legacy in choices, f"{model_id} dropped legacy {legacy!r}"
+
     @pytest.mark.parametrize("model_id", [
         # Claude 3.5 and 3 don't support extended thinking
         "us.anthropic.claude-3-5-haiku-20241022-v1:0",


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org