You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2025/10/17 17:40:02 UTC

(tooling-trusted-releases) branch main updated: Add a basic OSV command to the SBOM tool

This is an automated email from the ASF dual-hosted git repository.

sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git


The following commit(s) were added to refs/heads/main by this push:
     new 7bbb22e  Add a basic OSV command to the SBOM tool
7bbb22e is described below

commit 7bbb22e00494948817499f12f7ce05f68e43635c
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Fri Oct 17 18:39:42 2025 +0100

    Add a basic OSV command to the SBOM tool
---
 .gitignore                              |   1 +
 atr/sbom/cli.py                         |  19 +++-
 atr/sbom/models/__init__.py             |   3 +-
 atr/sbom/models/{__init__.py => osv.py} |  23 ++--
 atr/sbom/osv.py                         | 192 ++++++++++++++++++++++++++++++++
 5 files changed, 224 insertions(+), 14 deletions(-)

diff --git a/.gitignore b/.gitignore
index e55fc1a..0c7bf91 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 __pycache__/
 .DS_Store
 .env
+.hypothesis/
 .mypy_cache/
 .pytest_cache/
 .ruff_cache/
diff --git a/atr/sbom/cli.py b/atr/sbom/cli.py
index e6e0ce9..cd80cab 100644
--- a/atr/sbom/cli.py
+++ b/atr/sbom/cli.py
@@ -23,7 +23,7 @@ import sys
 
 import yyjson
 
-from . import models
+from . import models, osv
 from .conformance import ntia_2021_issues
 from .cyclonedx import validate_cli, validate_py
 from .licenses import check
@@ -71,6 +71,19 @@ def command_missing(bundle: models.bundle.Bundle) -> None:
         print(error)
 
 
+def command_osv(bundle: models.bundle.Bundle) -> None:
+    results, ignored_count = asyncio.run(osv.scan_bundle(bundle))
+    if ignored_count > 0:
+        print(f"Warning: {ignored_count} components ignored (missing purl or version)")
+    for component_result in results:
+        print(component_result.purl)
+        for vuln in component_result.vulnerabilities:
+            vuln_id = vuln.get("id", "unknown")
+            modified = vuln.get("modified", "")
+            summary = vuln.get("summary", "(no summary)")
+            print(f"  {vuln_id} {modified} {summary}")
+
+
 def command_outdated(bundle: models.bundle.Bundle) -> None:
     outdated = plugin_outdated_version(bundle.bom)
     if outdated:
@@ -140,7 +153,7 @@ def command_where(bundle: models.bundle.Bundle) -> None:
                     print()
 
 
-def main() -> None:
+def main() -> None:  # noqa: C901
     path = pathlib.Path(sys.argv[2])
     bundle = path_to_bundle(path)
     match sys.argv[1]:
@@ -150,6 +163,8 @@ def main() -> None:
             command_merge(bundle)
         case "missing":
             command_missing(bundle)
+        case "osv":
+            command_osv(bundle)
         case "outdated":
             command_outdated(bundle)
         case "patch":
diff --git a/atr/sbom/models/__init__.py b/atr/sbom/models/__init__.py
index 7077d15..fd44fb8 100644
--- a/atr/sbom/models/__init__.py
+++ b/atr/sbom/models/__init__.py
@@ -17,7 +17,7 @@
 
 from __future__ import annotations
 
-from . import base, bom, bundle, conformance, licenses, maven, patch, sbomqs
+from . import base, bom, bundle, conformance, licenses, maven, osv, patch, sbomqs
 
 __all__ = [
     "base",
@@ -26,6 +26,7 @@ __all__ = [
     "conformance",
     "licenses",
     "maven",
+    "osv",
     "patch",
     "sbomqs",
 ]
diff --git a/atr/sbom/models/__init__.py b/atr/sbom/models/osv.py
similarity index 76%
copy from atr/sbom/models/__init__.py
copy to atr/sbom/models/osv.py
index 7077d15..30cf8aa 100644
--- a/atr/sbom/models/__init__.py
+++ b/atr/sbom/models/osv.py
@@ -17,15 +17,16 @@
 
 from __future__ import annotations
 
-from . import base, bom, bundle, conformance, licenses, maven, patch, sbomqs
+from typing import Any
 
-__all__ = [
-    "base",
-    "bom",
-    "bundle",
-    "conformance",
-    "licenses",
-    "maven",
-    "patch",
-    "sbomqs",
-]
+from .base import Lax
+
+
+class QueryResult(Lax):
+    vulns: list[dict[str, Any]] | None = None
+    next_page_token: str | None = None
+
+
+class ComponentVulnerabilities(Lax):
+    purl: str
+    vulnerabilities: list[dict[str, Any]]
diff --git a/atr/sbom/osv.py b/atr/sbom/osv.py
new file mode 100644
index 0000000..f6b80cc
--- /dev/null
+++ b/atr/sbom/osv.py
@@ -0,0 +1,192 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import aiohttp
+
+from . import models
+
+_DEBUG: bool = os.environ.get("DEBUG_SBOM_TOOL") == "1"
+_OSV_API_BASE: str = "https://api.osv.dev/v1"
+
+
+async def scan_bundle(bundle: models.bundle.Bundle) -> tuple[list[models.osv.ComponentVulnerabilities], int]:
+    components = bundle.bom.components or []
+    queries, ignored_count = _scan_bundle_build_queries(components)
+    if _DEBUG:
+        print(f"[DEBUG] Scanning {len(queries)} components for vulnerabilities")
+        if ignored_count > 0:
+            print(f"[DEBUG] {ignored_count} components ignored (missing purl or version)")
+    async with aiohttp.ClientSession() as session:
+        component_vulns_map = await _scan_bundle_fetch_vulnerabilities(session, queries, 1000)
+        if _DEBUG:
+            print(f"[DEBUG] Total components with vulnerabilities: {len(component_vulns_map)}")
+        await _scan_bundle_populate_vulnerabilities(session, component_vulns_map)
+    result: list[models.osv.ComponentVulnerabilities] = []
+    for purl, vulns in component_vulns_map.items():
+        result.append(models.osv.ComponentVulnerabilities(purl=purl, vulnerabilities=vulns))
+    return result, ignored_count
+
+
+def _component_purl_with_version(component: models.bom.Component) -> str | None:
+    if component.purl is None:
+        return None
+    if component.version is None:
+        return None
+    version = component.version.strip()
+    if not version:
+        return None
+    purl = component.purl
+    split_index = len(purl)
+    question_index = purl.find("?")
+    if (question_index != -1) and (question_index < split_index):
+        split_index = question_index
+    hash_index = purl.find("#")
+    if (hash_index != -1) and (hash_index < split_index):
+        split_index = hash_index
+    if "@" in purl[:split_index]:
+        return purl
+    base = purl[:split_index]
+    suffix = purl[split_index:]
+    return f"{base}@{version}{suffix}"
+
+
+async def _fetch_vulnerabilities_for_batch(
+    session: aiohttp.ClientSession,
+    queries: list[dict[str, Any]],
+) -> list[models.osv.QueryResult]:
+    if _DEBUG:
+        print(f"[DEBUG] Sending querybatch with {len(queries)} queries")
+    payload = {"queries": queries}
+    async with session.post(f"{_OSV_API_BASE}/querybatch", json=payload) as response:
+        # TODO: Should we retry?
+        response.raise_for_status()
+        data = await response.json()
+    results_data = data.get("results", [])
+    if _DEBUG:
+        print(f"[DEBUG] Received {len(results_data)} results")
+    return [models.osv.QueryResult.model_validate(result) for result in results_data]
+
+
+async def _fetch_vulnerability_details(
+    session: aiohttp.ClientSession,
+    vuln_id: str,
+) -> dict[str, Any]:
+    if _DEBUG:
+        print(f"[DEBUG] Fetching details for {vuln_id}")
+    async with session.get(f"{_OSV_API_BASE}/vulns/{vuln_id}") as response:
+        response.raise_for_status()
+        return await response.json()
+
+
+async def _paginate_query(
+    session: aiohttp.ClientSession,
+    query: dict[str, Any],
+    page_token: str,
+) -> list[dict[str, Any]]:
+    all_vulns: list[dict[str, Any]] = []
+    current_query = query.copy()
+    current_query["page_token"] = page_token
+    page = 0
+    while True:
+        page += 1
+        if _DEBUG and page > 1:
+            print(f"[DEBUG] Paginating query (page {page})")
+        results = await _fetch_vulnerabilities_for_batch(session, [current_query])
+        if not results:
+            break
+        result = results[0]
+        if result.vulns:
+            all_vulns.extend(result.vulns)
+        next_page_token = result.next_page_token
+        if next_page_token is None:
+            break
+        current_query["page_token"] = next_page_token
+    return all_vulns
+
+
+def _scan_bundle_build_queries(
+    components: list[models.bom.Component],
+) -> tuple[list[tuple[str, dict[str, Any]]], int]:
+    queries: list[tuple[str, dict[str, Any]]] = []
+    ignored_count = 0
+    for component in components:
+        purl_with_version = _component_purl_with_version(component)
+        if purl_with_version is None:
+            ignored_count += 1
+            continue
+        query = {"package": {"purl": purl_with_version}}
+        queries.append((purl_with_version, query))
+    return queries, ignored_count
+
+
+async def _scan_bundle_fetch_vulnerabilities(
+    session: aiohttp.ClientSession,
+    queries: list[tuple[str, dict[str, Any]]],
+    batch_size: int,
+) -> dict[str, list[dict[str, Any]]]:
+    component_vulns_map: dict[str, list[dict[str, Any]]] = {}
+    for batch_start in range(0, len(queries), batch_size):
+        batch_end = min(batch_start + batch_size, len(queries))
+        batch = queries[batch_start:batch_end]
+        if _DEBUG:
+            batch_num = batch_start // batch_size + 1
+            print(f"[DEBUG] Processing batch {batch_num} ({batch_start + 1}-{batch_end}/{len(queries)})")
+        batch_queries = [query for _purl, query in batch]
+        batch_results = await _fetch_vulnerabilities_for_batch(session, batch_queries)
+        if _DEBUG and (len(batch_results) != len(batch)):
+            print(f"[DEBUG] count mismatch (expected {len(batch)}, got {len(batch_results)})")
+        for i, (purl, query) in enumerate(batch):
+            if i >= len(batch_results):
+                break
+            query_result = batch_results[i]
+            if query_result.vulns:
+                existing_vulns = component_vulns_map.setdefault(purl, [])
+                existing_vulns.extend(query_result.vulns)
+                if _DEBUG:
+                    print(f"[DEBUG] {purl}: {len(query_result.vulns)} vulnerabilities")
+            if query_result.next_page_token:
+                if _DEBUG:
+                    print(f"[DEBUG] {purl}: has pagination, fetching remaining pages")
+                existing_vulns = component_vulns_map.setdefault(purl, [])
+                paginated = await _paginate_query(session, query, query_result.next_page_token)
+                existing_vulns.extend(paginated)
+    return component_vulns_map
+
+
+async def _scan_bundle_populate_vulnerabilities(
+    session: aiohttp.ClientSession,
+    component_vulns_map: dict[str, list[dict[str, Any]]],
+) -> None:
+    details_cache: dict[str, dict[str, Any]] = {}
+    for vulns in component_vulns_map.values():
+        for vuln in vulns:
+            vuln_id = vuln.get("id")
+            if not vuln_id:
+                continue
+            details = details_cache.get(vuln_id)
+            if details is None:
+                details = await _fetch_vulnerability_details(session, vuln_id)
+                details_cache[vuln_id] = details
+            vuln.clear()
+            vuln.update(details)
+    if _DEBUG:
+        print(f"[DEBUG] Fetched details for {len(details_cache)} unique vulnerabilities")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org