You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ak...@apache.org on 2026/06/09 00:59:16 UTC
(tooling-gofannon) 01/04: bedrock-mythos: Mantle-backed provider for Mythos preview
This is an automated email from the ASF dual-hosted git repository.
andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-gofannon.git
commit e2046468a8d71df893f9296bd6bc7b090f4f3cb4
Author: Andrew Musselman <ak...@apache.org>
AuthorDate: Mon Jun 8 21:53:30 2026 +0000
bedrock-mythos: Mantle-backed provider for Mythos preview
Adds the bedrock-mythos provider routing Mythos preview model calls
through AWS Bedrock Mantle (the OpenAI-compatible Bedrock endpoint at
https://bedrock-mantle.{region}.api.aws/v1) in a dedicated Anthropic-
provided AWS account.
Auth flow:
1. boto3 picks up ambient credentials (EC2 instance profile in prod,
'aws login' session locally)
2. STS AssumeRole into arn:aws:iam::861792231409:role/bedrock-devs
3. aws-bedrock-token-generator mints a SigV4-derived bearer token
from the assumed credentials (no env mutation; credentials passed
explicitly via botocore.credentials.Credentials)
4. Bearer token passed to litellm as api_key, region passed as
aws_region_name (without it litellm defaults to us-east-1)
5. litellm routes through bedrock_mantle/<model-id> to the Mantle
endpoint in ap-southeast-4
Token cached in-process per role ARN; refreshed 5 min before the
underlying STS credentials expire to avoid mid-request expiration.
Both call_llm and stream_llm paths updated. boto3 and the token
generator are imported lazily inside the helper to avoid forcing the
dependencies on module load for installations that don't use this
provider.
Configuration is all non-secret; lives in provider_config rather than
puppet eyaml. The Service Role we provisioned in account 861792231409
(arn:aws:iam::861792231409:role/bedrock-service-role) is registered
with Bedrock at the account level — no request-level passing needed
for Mantle inference.
Verified locally: litellm 1.83.0 (current pin: >=1.77.5) recognizes
the bedrock_mantle/ prefix and dispatches to the Mantle handler.
Cross-account assume-role plumbing has been verified end-to-end from
a developer laptop. The cross-account path on the gofannon EC2 host
(INFRA-27980 in account 534396592182 assuming bedrock-devs) is
unverified until first deploy — trust policy is in place, awaiting
a real call.
---
.../user-service/config/bedrock_mythos/__init__.py | 45 ++++++++
.../api/user-service/config/provider_config.py | 22 ++++
webapp/packages/api/user-service/requirements.txt | 1 +
.../api/user-service/services/llm_service.py | 115 ++++++++++++++++++++-
4 files changed, 181 insertions(+), 2 deletions(-)
diff --git a/webapp/packages/api/user-service/config/bedrock_mythos/__init__.py b/webapp/packages/api/user-service/config/bedrock_mythos/__init__.py
new file mode 100644
index 0000000..6dc774b
--- /dev/null
+++ b/webapp/packages/api/user-service/config/bedrock_mythos/__init__.py
@@ -0,0 +1,45 @@
+# Mythos preview model configuration (accessed via AWS Bedrock Mantle).
+#
+# Mantle is a distinct Bedrock endpoint
+# (https://bedrock-mantle.{region}.api.aws/v1) with OpenAI-compatible
+# APIs and its own IAM action namespace (bedrock-mantle:*). Mythos is
+# exposed there as anthropic.claude-mythos-preview.
+#
+# Auth flow at request time:
+# 1. boto3 reads ambient credentials (EC2 instance profile in prod,
+# `aws login` session locally)
+# 2. STS AssumeRole into the cross-account devs role in Anthropic's
+# preview AWS account (861792231409)
+# 3. aws-bedrock-token-generator mints a SigV4-derived bearer token
+# from the assumed credentials
+# 4. Bearer token passed to litellm as api_key; litellm routes via
+# its bedrock_mantle provider to the Mantle endpoint
+#
+# See llm_service._get_bedrock_mythos_api_key() and the bedrock-mythos
+# entry in provider_config.PROVIDER_CONFIG.
+
+models = {
+ "anthropic.claude-mythos-preview": {
+ "returns_thoughts": True,
+ "supports_effort": True,
+ "supports_thinking": True,
+ "context_window": 1000000,
+ "parameters": {
+ "temperature": {
+ "type": "float",
+ "default": 1.0,
+ "min": 0.0,
+ "max": 1.0,
+ "description": "Randomness (0=focused, 1=creative). Locked to 1.0 when thinking enabled.",
+ "mutually_exclusive_with": ["top_p"],
+ },
+ "max_tokens": {
+ "type": "integer",
+ "default": 4096,
+ "min": 1,
+ "max": 131072,
+ "description": "Maximum tokens to generate (up to 128K output).",
+ },
+ },
+ },
+}
diff --git a/webapp/packages/api/user-service/config/provider_config.py b/webapp/packages/api/user-service/config/provider_config.py
index 56ecee3..14b72be 100644
--- a/webapp/packages/api/user-service/config/provider_config.py
+++ b/webapp/packages/api/user-service/config/provider_config.py
@@ -1,5 +1,6 @@
from .anthropic import models as anthropic_models
from .bedrock import models as bedrock_models
+from .bedrock_mythos import models as bedrock_mythos_models
from .gemini import models as gemini_models
from .openai import models as openai_models
from .openrouter import models as openrouter_models
@@ -25,6 +26,27 @@ PROVIDER_CONFIG = {
"api_key_env_var": "AWS_BEARER_TOKEN_BEDROCK",
"models": bedrock_models
},
+ # Mythos preview — Bedrock Mantle endpoint in Anthropic's preview
+ # AWS account, accessed via cross-account STS AssumeRole + a SigV4-
+ # derived bearer token. No static credentials anywhere; the bearer
+ # token is minted from short-lived assumed-role credentials at
+ # request time, then passed to litellm as api_key. The region is
+ # also injected so litellm constructs the right Mantle endpoint
+ # URL (bedrock-mantle.{region}.api.aws/v1) — without it litellm
+ # defaults to us-east-1.
+ #
+ # Config knobs:
+ # aws_region — region where Mythos is provisioned
+ # assume_role_arn — cross-account devs role to assume
+ # litellm_provider_prefix — built as "bedrock_mantle/<model-id>"
+ # so litellm routes through its Mantle
+ # provider (OpenAI-compatible endpoint)
+ "bedrock-mythos": {
+ "aws_region": "ap-southeast-4",
+ "assume_role_arn": "arn:aws:iam::861792231409:role/bedrock-devs",
+ "litellm_provider_prefix": "bedrock_mantle",
+ "models": bedrock_mythos_models,
+ },
"openrouter": {
"api_key_env_var": "OPENROUTER_API_KEY",
"models": openrouter_models,
diff --git a/webapp/packages/api/user-service/requirements.txt b/webapp/packages/api/user-service/requirements.txt
index 7e9f46e..8f57c72 100644
--- a/webapp/packages/api/user-service/requirements.txt
+++ b/webapp/packages/api/user-service/requirements.txt
@@ -23,3 +23,4 @@ cffi
a2wsgi
google-cloud-logging
ldap3>=2.9
+aws-bedrock-token-generator>=1.0
diff --git a/webapp/packages/api/user-service/services/llm_service.py b/webapp/packages/api/user-service/services/llm_service.py
index 9c40914..1fc2e20 100644
--- a/webapp/packages/api/user-service/services/llm_service.py
+++ b/webapp/packages/api/user-service/services/llm_service.py
@@ -4,6 +4,11 @@ import os
import random
from config import settings
from config.provider_config import PROVIDER_CONFIG
+# boto3 and aws-bedrock-token-generator are imported lazily inside
+# _get_bedrock_mythos_api_key to avoid forcing those deps onto module
+# load when the bedrock-mythos provider isn't configured. boto3 is
+# already transitively present via storage_service / dynamodb /
+# observability_service.
from services.database_service import get_database_service
from services.user_service import get_user_service
from typing import Any, AsyncGenerator, Dict, List, Tuple, Optional
@@ -68,6 +73,70 @@ def _is_opus_4_7_or_later(model: str) -> bool:
return any(tag in model for tag in ("claude-opus-4-7", "claude-opus-4-8"))
+# In-process cache of Mantle bearer tokens, keyed by role ARN.
+# STS AssumeRole defaults to 1-hour sessions and bearer-token validity
+# is bounded by the credentials' lifetime, so we refresh ~5 min before
+# expiry to avoid mid-request expiration.
+_BEDROCK_MYTHOS_TOKEN_CACHE: Dict[str, Dict[str, Any]] = {}
+_TOKEN_REFRESH_MARGIN_SECONDS = 300
+
+
+def _get_bedrock_mythos_api_key(
+ assume_role_arn: str,
+ region: str,
+) -> str:
+ """Return a Mantle bearer token for the bedrock-mythos provider.
+
+ Performs an STS AssumeRole into the cross-account devs role in
+ Anthropic's preview AWS account, then derives a SigV4 bearer token
+ from those short-lived credentials. The bearer token is what
+ litellm passes to the bedrock-mantle endpoint as the Authorization
+ header.
+
+ Cached in-process per role ARN; refreshes when within
+ `_TOKEN_REFRESH_MARGIN_SECONDS` of credentials expiry.
+
+ Returns the bearer token string, suitable for passing to litellm
+ as the api_key kwarg on a bedrock_mantle/* request.
+ """
+ import boto3
+ from botocore.credentials import Credentials
+ from aws_bedrock_token_generator import BedrockTokenGenerator
+ from datetime import datetime, timezone
+
+ now = datetime.now(timezone.utc)
+ cached = _BEDROCK_MYTHOS_TOKEN_CACHE.get(assume_role_arn)
+ if cached:
+ if (cached["_expires_at"] - now).total_seconds() > _TOKEN_REFRESH_MARGIN_SECONDS:
+ return cached["token"]
+
+ # Assume the cross-account role using whatever credentials are
+ # ambient (EC2 instance profile in prod, aws login locally).
+ sts = boto3.client("sts")
+ response = sts.assume_role(
+ RoleArn=assume_role_arn,
+ RoleSessionName="gofannon-bedrock-mythos",
+ )
+ raw = response["Credentials"]
+
+ # Wrap the assumed credentials as a botocore Credentials object
+ # so we can pass them explicitly to BedrockTokenGenerator without
+ # mutating environment variables (which would not be thread-safe
+ # under the asyncio worker pool).
+ creds = Credentials(
+ access_key=raw["AccessKeyId"],
+ secret_key=raw["SecretAccessKey"],
+ token=raw["SessionToken"],
+ )
+ token = BedrockTokenGenerator().get_token(creds, region=region)
+
+ _BEDROCK_MYTHOS_TOKEN_CACHE[assume_role_arn] = {
+ "token": token,
+ "_expires_at": raw["Expiration"],
+ }
+ return token
+
+
async def call_llm(
provider: str,
model: str,
@@ -90,7 +159,13 @@ async def call_llm(
model_config = PROVIDER_CONFIG.get(provider, {}).get("models", {}).get(model, {})
api_style = model_config.get("api_style")
- model_string = f"{provider}/{model}"
+ # If the provider config supplies an explicit litellm prefix (e.g.
+ # bedrock-mythos -> "bedrock_mantle"), use it so litellm routes
+ # through its native handler for that backend. Defaults to the
+ # gofannon provider key for backwards compatibility.
+ provider_cfg = PROVIDER_CONFIG.get(provider, {})
+ litellm_prefix = provider_cfg.get("litellm_provider_prefix", provider)
+ model_string = f"{litellm_prefix}/{model}"
thoughts = None
content = ""
@@ -129,6 +204,21 @@ async def call_llm(
if api_key:
kwargs["api_key"] = api_key
+ # bedrock-mythos: mint a short-lived Mantle bearer token at
+ # request time, pass it to litellm as api_key. Also pass the
+ # region — litellm uses it to construct the Mantle endpoint URL
+ # (bedrock-mantle.{region}.api.aws/v1); without it litellm
+ # defaults to us-east-1 which doesn't host Mythos.
+ # The bearer token overrides any user-specific key above since
+ # bedrock-mythos auth is account-scoped, not per-user. See
+ # _get_bedrock_mythos_api_key().
+ if provider_cfg.get("assume_role_arn"):
+ kwargs["api_key"] = _get_bedrock_mythos_api_key(
+ assume_role_arn=provider_cfg["assume_role_arn"],
+ region=provider_cfg["aws_region"],
+ )
+ kwargs["aws_region_name"] = provider_cfg["aws_region"]
+
# Only use aresponses API when we actually need its features (tools or reasoning)
# Otherwise use standard acompletion which is more reliable
@@ -511,7 +601,13 @@ async def stream_llm(
Note: Cost tracking is not available for streaming responses.
"""
- model_string = f"{provider}/{model}"
+ # If the provider config supplies an explicit litellm prefix (e.g.
+ # bedrock-mythos -> "bedrock_mantle"), use it so litellm routes
+ # through its native handler for that backend. Defaults to the
+ # gofannon provider key for backwards compatibility.
+ provider_cfg = PROVIDER_CONFIG.get(provider, {})
+ litellm_prefix = provider_cfg.get("litellm_provider_prefix", provider)
+ model_string = f"{litellm_prefix}/{model}"
# Filter out None values from parameters
filtered_params = {k: v for k, v in parameters.items() if v is not None}
@@ -543,6 +639,21 @@ async def stream_llm(
if api_key:
kwargs["api_key"] = api_key
+ # bedrock-mythos: mint a short-lived Mantle bearer token at
+ # request time, pass it to litellm as api_key. Also pass the
+ # region — litellm uses it to construct the Mantle endpoint URL
+ # (bedrock-mantle.{region}.api.aws/v1); without it litellm
+ # defaults to us-east-1 which doesn't host Mythos.
+ # The bearer token overrides any user-specific key above since
+ # bedrock-mythos auth is account-scoped, not per-user. See
+ # _get_bedrock_mythos_api_key().
+ if provider_cfg.get("assume_role_arn"):
+ kwargs["api_key"] = _get_bedrock_mythos_api_key(
+ assume_role_arn=provider_cfg["assume_role_arn"],
+ region=provider_cfg["aws_region"],
+ )
+ kwargs["aws_region_name"] = provider_cfg["aws_region"]
+
try:
response = await litellm.acompletion(**kwargs)
async for chunk in response:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org