You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/07/29 14:46:48 UTC
(tooling-trusted-releases) branch sbp updated: Retry OSV requests
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/sbp by this push:
new 96460923 Retry OSV requests
96460923 is described below
commit 964609233ec0dfb7b47f0fa2be03b1cac8e4c580
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Wed Jul 29 15:46:25 2026 +0100
Retry OSV requests
---
atr/sbom/osv.py | 53 +++++++++++++++++----
tests/unit/test_sbom_osv.py | 111 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 156 insertions(+), 8 deletions(-)
diff --git a/atr/sbom/osv.py b/atr/sbom/osv.py
index 36833f7e..e233d6eb 100644
--- a/atr/sbom/osv.py
+++ b/atr/sbom/osv.py
@@ -17,6 +17,7 @@
from __future__ import annotations
+import asyncio
import os
from typing import TYPE_CHECKING, Any, Final
@@ -34,6 +35,7 @@ if TYPE_CHECKING:
_DEBUG: bool = os.environ.get("DEBUG_SBOM_TOOL") == "1"
_OSV_API_BASE: Final[str] = "https://api.osv.dev/v1"
_API_TIMEOUT: Final = aiohttp.ClientTimeout(total=60, connect=10)
+_RETRY_DELAY_SECONDS: Final = (1, 3)
_SOURCE_DATABASE_NAMES: Final = {
"ASB": "Android Security Bulletin",
"PUB": "Android Security Bulletin",
@@ -197,10 +199,7 @@ async def _fetch_vulnerabilities_for_batch(
if _DEBUG:
print(f"[DEBUG] Sending querybatch with {len(queries)} queries")
payload = {"queries": queries}
- async with session.post(f"{_OSV_API_BASE}/querybatch", json=payload) as response:
- # TODO: Should we retry?
- response.raise_for_status()
- data = await response.json()
+ data = await _request_json(session, "POST", f"{_OSV_API_BASE}/querybatch", payload)
results_data = data.get("results", [])
if _DEBUG:
print(f"[DEBUG] Received {len(results_data)} results")
@@ -213,10 +212,8 @@ async def _fetch_vulnerability_details(
) -> models.osv.VulnerabilityDetails:
if _DEBUG:
print(f"[DEBUG] Fetching details for {vuln_id}")
- async with session.get(f"{_OSV_API_BASE}/vulns/{vuln_id}") as response:
- response.raise_for_status()
- data = await response.json()
- return models.osv.VulnerabilityDetails.model_validate(data)
+ data = await _request_json(session, "GET", f"{_OSV_API_BASE}/vulns/{vuln_id}")
+ return models.osv.VulnerabilityDetails.model_validate(data)
def _get_source(vuln: models.osv.VulnerabilityDetails) -> dict[str, str]:
@@ -257,6 +254,46 @@ async def _paginate_query(
return all_vulns
+async def _request_json(
+ session: aiohttp.ClientSession,
+ method: str,
+ url: str,
+ payload: dict[str, Any] | None = None,
+) -> Any:
+ for delay in _RETRY_DELAY_SECONDS:
+ try:
+ return await _request_json_once(session, method, url, payload)
+ except (TimeoutError, aiohttp.ClientConnectionError, aiohttp.ClientPayloadError):
+ await asyncio.sleep(delay)
+ except aiohttp.ClientResponseError as e:
+ if (e.status != 429) and (e.status < 500):
+ raise
+ if _retry_after_exceeds_delay(e, delay):
+ raise
+ await asyncio.sleep(delay)
+ return await _request_json_once(session, method, url, payload)
+
+
+async def _request_json_once(
+ session: aiohttp.ClientSession,
+ method: str,
+ url: str,
+ payload: dict[str, Any] | None,
+) -> Any:
+ async with session.request(method, url, json=payload) as response:
+ response.raise_for_status()
+ return await response.json()
+
+
+def _retry_after_exceeds_delay(error: aiohttp.ClientResponseError, delay: int) -> bool:
+ if (error.status != 429) or (error.headers is None):
+ return False
+ retry_after = error.headers.get("Retry-After")
+ if retry_after is None:
+ return False
+ return (not retry_after.isdigit()) or (int(retry_after) > delay)
+
+
def _scan_bundle_build_queries(
components: list[Component],
) -> tuple[list[tuple[str, dict[str, Any]]], list[str]]:
diff --git a/tests/unit/test_sbom_osv.py b/tests/unit/test_sbom_osv.py
new file mode 100644
index 00000000..503aee2a
--- /dev/null
+++ b/tests/unit/test_sbom_osv.py
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import aiohttp
+import pytest
+
+import atr.sbom.osv as osv
+
+
+class FakeResponse:
+ def __init__(self, status: int, payload=None, headers=None):
+ self.status = status
+ self._payload = payload
+ self._headers = headers
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, exc_type, exc, tb):
+ return False
+
+ async def json(self, content_type=None):
+ if isinstance(self._payload, Exception):
+ raise self._payload
+ return self._payload
+
+ def raise_for_status(self) -> None:
+ if self.status >= 400:
+ raise aiohttp.ClientResponseError(None, (), status=self.status, headers=self._headers)
+
+
+class FakeSession:
+ def __init__(self, responses):
+ self._responses = list(responses)
+ self.requests = 0
+
+ def request(self, method, url, json=None):
+ self.requests += 1
+ response = self._responses.pop(0)
+ if isinstance(response, Exception):
+ raise response
+ return response
+
+
+@pytest.fixture
+def sleeps(monkeypatch):
+ recorded = []
+
+ async def fake_sleep(delay):
+ recorded.append(delay)
+
+ monkeypatch.setattr(osv.asyncio, "sleep", fake_sleep)
+ return recorded
+
+
+async def test_request_json_does_not_retry_client_errors(sleeps):
+ session = FakeSession([FakeResponse(404)])
+
+ with pytest.raises(aiohttp.ClientResponseError):
+ await osv._request_json(session, "GET", "https://osv.test/vulns/x")
+ assert session.requests == 1
+ assert sleeps == []
+
+
+async def test_request_json_gives_up_after_exhausting_retries(sleeps):
+ session = FakeSession([aiohttp.ClientConnectionError(), TimeoutError(), FakeResponse(503)])
+
+ with pytest.raises(aiohttp.ClientResponseError):
+ await osv._request_json(session, "GET", "https://osv.test/vulns/x")
+ assert session.requests == 3
+ assert sleeps == list(osv._RETRY_DELAY_SECONDS)
+
+
+async def test_request_json_retries_server_errors(sleeps):
+ session = FakeSession([FakeResponse(500), FakeResponse(429, headers={"Retry-After": "3"}), FakeResponse(200, {})])
+
+ result = await osv._request_json(session, "POST", "https://osv.test/querybatch", {"queries": []})
+ assert result == {}
+ assert session.requests == 3
+ assert sleeps == list(osv._RETRY_DELAY_SECONDS)
+
+
+async def test_request_json_retries_truncated_responses(sleeps):
+ session = FakeSession([FakeResponse(200, aiohttp.ClientPayloadError()), FakeResponse(200, {})])
+
+ assert await osv._request_json(session, "GET", "https://osv.test/vulns/x") == {}
+ assert session.requests == 2
+ assert sleeps == [osv._RETRY_DELAY_SECONDS[0]]
+
+
+async def test_request_json_stops_when_retry_after_exceeds_delay(sleeps):
+ session = FakeSession([FakeResponse(429, headers={"Retry-After": "2"})])
+
+ with pytest.raises(aiohttp.ClientResponseError):
+ await osv._request_json(session, "GET", "https://osv.test/vulns/x")
+ assert session.requests == 1
+ assert sleeps == []
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org