You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2025/10/17 17:40:02 UTC
(tooling-trusted-releases) branch main updated: Add a basic OSV command to the SBOM tool
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/main by this push:
new 7bbb22e Add a basic OSV command to the SBOM tool
7bbb22e is described below
commit 7bbb22e00494948817499f12f7ce05f68e43635c
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Fri Oct 17 18:39:42 2025 +0100
Add a basic OSV command to the SBOM tool
---
.gitignore | 1 +
atr/sbom/cli.py | 19 +++-
atr/sbom/models/__init__.py | 3 +-
atr/sbom/models/{__init__.py => osv.py} | 23 ++--
atr/sbom/osv.py | 192 ++++++++++++++++++++++++++++++++
5 files changed, 224 insertions(+), 14 deletions(-)
diff --git a/.gitignore b/.gitignore
index e55fc1a..0c7bf91 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
__pycache__/
.DS_Store
.env
+.hypothesis/
.mypy_cache/
.pytest_cache/
.ruff_cache/
diff --git a/atr/sbom/cli.py b/atr/sbom/cli.py
index e6e0ce9..cd80cab 100644
--- a/atr/sbom/cli.py
+++ b/atr/sbom/cli.py
@@ -23,7 +23,7 @@ import sys
import yyjson
-from . import models
+from . import models, osv
from .conformance import ntia_2021_issues
from .cyclonedx import validate_cli, validate_py
from .licenses import check
@@ -71,6 +71,19 @@ def command_missing(bundle: models.bundle.Bundle) -> None:
print(error)
+def command_osv(bundle: models.bundle.Bundle) -> None:
+ results, ignored_count = asyncio.run(osv.scan_bundle(bundle))
+ if ignored_count > 0:
+ print(f"Warning: {ignored_count} components ignored (missing purl or version)")
+ for component_result in results:
+ print(component_result.purl)
+ for vuln in component_result.vulnerabilities:
+ vuln_id = vuln.get("id", "unknown")
+ modified = vuln.get("modified", "")
+ summary = vuln.get("summary", "(no summary)")
+ print(f" {vuln_id} {modified} {summary}")
+
+
def command_outdated(bundle: models.bundle.Bundle) -> None:
outdated = plugin_outdated_version(bundle.bom)
if outdated:
@@ -140,7 +153,7 @@ def command_where(bundle: models.bundle.Bundle) -> None:
print()
-def main() -> None:
+def main() -> None: # noqa: C901
path = pathlib.Path(sys.argv[2])
bundle = path_to_bundle(path)
match sys.argv[1]:
@@ -150,6 +163,8 @@ def main() -> None:
command_merge(bundle)
case "missing":
command_missing(bundle)
+ case "osv":
+ command_osv(bundle)
case "outdated":
command_outdated(bundle)
case "patch":
diff --git a/atr/sbom/models/__init__.py b/atr/sbom/models/__init__.py
index 7077d15..fd44fb8 100644
--- a/atr/sbom/models/__init__.py
+++ b/atr/sbom/models/__init__.py
@@ -17,7 +17,7 @@
from __future__ import annotations
-from . import base, bom, bundle, conformance, licenses, maven, patch, sbomqs
+from . import base, bom, bundle, conformance, licenses, maven, osv, patch, sbomqs
__all__ = [
"base",
@@ -26,6 +26,7 @@ __all__ = [
"conformance",
"licenses",
"maven",
+ "osv",
"patch",
"sbomqs",
]
diff --git a/atr/sbom/models/__init__.py b/atr/sbom/models/osv.py
similarity index 76%
copy from atr/sbom/models/__init__.py
copy to atr/sbom/models/osv.py
index 7077d15..30cf8aa 100644
--- a/atr/sbom/models/__init__.py
+++ b/atr/sbom/models/osv.py
@@ -17,15 +17,16 @@
from __future__ import annotations
-from . import base, bom, bundle, conformance, licenses, maven, patch, sbomqs
+from typing import Any
-__all__ = [
- "base",
- "bom",
- "bundle",
- "conformance",
- "licenses",
- "maven",
- "patch",
- "sbomqs",
-]
+from .base import Lax
+
+
+class QueryResult(Lax):
+ vulns: list[dict[str, Any]] | None = None
+ next_page_token: str | None = None
+
+
+class ComponentVulnerabilities(Lax):
+ purl: str
+ vulnerabilities: list[dict[str, Any]]
diff --git a/atr/sbom/osv.py b/atr/sbom/osv.py
new file mode 100644
index 0000000..f6b80cc
--- /dev/null
+++ b/atr/sbom/osv.py
@@ -0,0 +1,192 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import aiohttp
+
+from . import models
+
+_DEBUG: bool = os.environ.get("DEBUG_SBOM_TOOL") == "1"
+_OSV_API_BASE: str = "https://api.osv.dev/v1"
+
+
+async def scan_bundle(bundle: models.bundle.Bundle) -> tuple[list[models.osv.ComponentVulnerabilities], int]:
+ components = bundle.bom.components or []
+ queries, ignored_count = _scan_bundle_build_queries(components)
+ if _DEBUG:
+ print(f"[DEBUG] Scanning {len(queries)} components for vulnerabilities")
+ if ignored_count > 0:
+ print(f"[DEBUG] {ignored_count} components ignored (missing purl or version)")
+ async with aiohttp.ClientSession() as session:
+ component_vulns_map = await _scan_bundle_fetch_vulnerabilities(session, queries, 1000)
+ if _DEBUG:
+ print(f"[DEBUG] Total components with vulnerabilities: {len(component_vulns_map)}")
+ await _scan_bundle_populate_vulnerabilities(session, component_vulns_map)
+ result: list[models.osv.ComponentVulnerabilities] = []
+ for purl, vulns in component_vulns_map.items():
+ result.append(models.osv.ComponentVulnerabilities(purl=purl, vulnerabilities=vulns))
+ return result, ignored_count
+
+
+def _component_purl_with_version(component: models.bom.Component) -> str | None:
+ if component.purl is None:
+ return None
+ if component.version is None:
+ return None
+ version = component.version.strip()
+ if not version:
+ return None
+ purl = component.purl
+ split_index = len(purl)
+ question_index = purl.find("?")
+ if (question_index != -1) and (question_index < split_index):
+ split_index = question_index
+ hash_index = purl.find("#")
+ if (hash_index != -1) and (hash_index < split_index):
+ split_index = hash_index
+ if "@" in purl[:split_index]:
+ return purl
+ base = purl[:split_index]
+ suffix = purl[split_index:]
+ return f"{base}@{version}{suffix}"
+
+
+async def _fetch_vulnerabilities_for_batch(
+ session: aiohttp.ClientSession,
+ queries: list[dict[str, Any]],
+) -> list[models.osv.QueryResult]:
+ if _DEBUG:
+ print(f"[DEBUG] Sending querybatch with {len(queries)} queries")
+ payload = {"queries": queries}
+ async with session.post(f"{_OSV_API_BASE}/querybatch", json=payload) as response:
+ # TODO: Should we retry?
+ response.raise_for_status()
+ data = await response.json()
+ results_data = data.get("results", [])
+ if _DEBUG:
+ print(f"[DEBUG] Received {len(results_data)} results")
+ return [models.osv.QueryResult.model_validate(result) for result in results_data]
+
+
+async def _fetch_vulnerability_details(
+ session: aiohttp.ClientSession,
+ vuln_id: str,
+) -> dict[str, Any]:
+ if _DEBUG:
+ print(f"[DEBUG] Fetching details for {vuln_id}")
+ async with session.get(f"{_OSV_API_BASE}/vulns/{vuln_id}") as response:
+ response.raise_for_status()
+ return await response.json()
+
+
+async def _paginate_query(
+ session: aiohttp.ClientSession,
+ query: dict[str, Any],
+ page_token: str,
+) -> list[dict[str, Any]]:
+ all_vulns: list[dict[str, Any]] = []
+ current_query = query.copy()
+ current_query["page_token"] = page_token
+ page = 0
+ while True:
+ page += 1
+ if _DEBUG and page > 1:
+ print(f"[DEBUG] Paginating query (page {page})")
+ results = await _fetch_vulnerabilities_for_batch(session, [current_query])
+ if not results:
+ break
+ result = results[0]
+ if result.vulns:
+ all_vulns.extend(result.vulns)
+ next_page_token = result.next_page_token
+ if next_page_token is None:
+ break
+ current_query["page_token"] = next_page_token
+ return all_vulns
+
+
+def _scan_bundle_build_queries(
+ components: list[models.bom.Component],
+) -> tuple[list[tuple[str, dict[str, Any]]], int]:
+ queries: list[tuple[str, dict[str, Any]]] = []
+ ignored_count = 0
+ for component in components:
+ purl_with_version = _component_purl_with_version(component)
+ if purl_with_version is None:
+ ignored_count += 1
+ continue
+ query = {"package": {"purl": purl_with_version}}
+ queries.append((purl_with_version, query))
+ return queries, ignored_count
+
+
+async def _scan_bundle_fetch_vulnerabilities(
+ session: aiohttp.ClientSession,
+ queries: list[tuple[str, dict[str, Any]]],
+ batch_size: int,
+) -> dict[str, list[dict[str, Any]]]:
+ component_vulns_map: dict[str, list[dict[str, Any]]] = {}
+ for batch_start in range(0, len(queries), batch_size):
+ batch_end = min(batch_start + batch_size, len(queries))
+ batch = queries[batch_start:batch_end]
+ if _DEBUG:
+ batch_num = batch_start // batch_size + 1
+ print(f"[DEBUG] Processing batch {batch_num} ({batch_start + 1}-{batch_end}/{len(queries)})")
+ batch_queries = [query for _purl, query in batch]
+ batch_results = await _fetch_vulnerabilities_for_batch(session, batch_queries)
+ if _DEBUG and (len(batch_results) != len(batch)):
+ print(f"[DEBUG] count mismatch (expected {len(batch)}, got {len(batch_results)})")
+ for i, (purl, query) in enumerate(batch):
+ if i >= len(batch_results):
+ break
+ query_result = batch_results[i]
+ if query_result.vulns:
+ existing_vulns = component_vulns_map.setdefault(purl, [])
+ existing_vulns.extend(query_result.vulns)
+ if _DEBUG:
+ print(f"[DEBUG] {purl}: {len(query_result.vulns)} vulnerabilities")
+ if query_result.next_page_token:
+ if _DEBUG:
+ print(f"[DEBUG] {purl}: has pagination, fetching remaining pages")
+ existing_vulns = component_vulns_map.setdefault(purl, [])
+ paginated = await _paginate_query(session, query, query_result.next_page_token)
+ existing_vulns.extend(paginated)
+ return component_vulns_map
+
+
+async def _scan_bundle_populate_vulnerabilities(
+ session: aiohttp.ClientSession,
+ component_vulns_map: dict[str, list[dict[str, Any]]],
+) -> None:
+ details_cache: dict[str, dict[str, Any]] = {}
+ for vulns in component_vulns_map.values():
+ for vuln in vulns:
+ vuln_id = vuln.get("id")
+ if not vuln_id:
+ continue
+ details = details_cache.get(vuln_id)
+ if details is None:
+ details = await _fetch_vulnerability_details(session, vuln_id)
+ details_cache[vuln_id] = details
+ vuln.clear()
+ vuln.update(details)
+ if _DEBUG:
+ print(f"[DEBUG] Fetched details for {len(details_cache)} unique vulnerabilities")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org