You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/09/10 14:01:54 UTC
(tooling-trusted-releases) branch sbp updated (7e06fda9 -> 907b3e49)
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a change to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
from 7e06fda9 Add a task for generating heatmaps
add e2e63f9c Tweak wording for lack of releases
add e0beac70 #1571 - json files for PMC and podling projects
add 227c4cff Add attic card into releases to group retired projects
new 907b3e49 Make SBOM file discovery more lenient
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
atr/analysis.py | 35 +++++-
atr/catalog_site.py | 185 ++++++++++++++++++++++++++++
atr/get/release.py | 28 +++--
atr/tasks/checks/paths.py | 7 +-
atr/templates/catalog_site/project.html | 6 +-
atr/templates/committee-view.html | 2 +-
tests/unit/test_catalog_site.py | 162 ++++++++++++++++++++++++
tests/unit/test_classify.py | 18 ++-
tests/unit/test_documentation_tree_check.py | 14 +++
tests/unit/test_releases_catalog.py | 34 ++++-
tests/unit/test_sbom_candidates.py | 53 ++++++++
11 files changed, 520 insertions(+), 24 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-trusted-releases) 01/01: Make SBOM file discovery more lenient
Posted by sb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 907b3e49ecc11a31f1ed180b43bf81f70893bd13
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Thu Sep 10 15:01:25 2026 +0100
Make SBOM file discovery more lenient
---
atr/analysis.py | 35 ++++++++++++++++---
atr/tasks/checks/paths.py | 7 ++--
tests/unit/test_classify.py | 18 +++++++---
tests/unit/test_documentation_tree_check.py | 14 ++++++++
tests/unit/test_sbom_candidates.py | 53 +++++++++++++++++++++++++++++
5 files changed, 116 insertions(+), 11 deletions(-)
diff --git a/atr/analysis.py b/atr/analysis.py
index f4aba48a..61b824e5 100755
--- a/atr/analysis.py
+++ b/atr/analysis.py
@@ -59,9 +59,25 @@ ARTIFACT_SUFFIXES: Final[list[str]] = [
"zip",
]
-CYCLONEDX_JSON_SUFFIXES: Final[tuple[str, ...]] = (".cdx.json", "-cyclonedx.json")
+CYCLONEDX_JSON_FILENAMES: Final[tuple[str, ...]] = ("sbom.json", "bom.json")
+
+CYCLONEDX_JSON_SUFFIXES: Final[tuple[str, ...]] = (
+ ".cdx.json",
+ "-cyclonedx.json",
+ ".cyclonedx.json",
+ "-sbom.json",
+ "-bom.json",
+)
+
+CYCLONEDX_XML_FILENAMES: Final[tuple[str, ...]] = ("sbom.xml", "bom.xml")
-CYCLONEDX_XML_SUFFIXES: Final[tuple[str, ...]] = (".cdx.xml", "-cyclonedx.xml")
+CYCLONEDX_XML_SUFFIXES: Final[tuple[str, ...]] = (
+ ".cdx.xml",
+ "-cyclonedx.xml",
+ ".cyclonedx.xml",
+ "-sbom.xml",
+ "-bom.xml",
+)
DISALLOWED_FILENAMES: Final[frozenset[str]] = frozenset(
{
@@ -170,6 +186,7 @@ VARIANT_PATTERNS: Final[list[str]] = [
_CANDIDATE_TAG: Final = r"(?: Candidate | candidate | RC | Rc | rc ) [.-]? [0-9]+"
_CANDIDATE_PARTIAL: Final = re.compile(rf"(?x) - {_CANDIDATE_TAG}")
_CANDIDATE_WHOLE: Final = re.compile(rf"(?x) ^ {_CANDIDATE_TAG} $")
+_SOURCE_CLASSIFIER: Final = re.compile(r"[_-](?:source-release|sources|source|src)(?=[_-][0-9]|$)")
@dataclasses.dataclass
@@ -368,11 +385,11 @@ def is_cyclonedx(name: str) -> bool:
def is_cyclonedx_json(name: str) -> bool:
- return name.endswith(CYCLONEDX_JSON_SUFFIXES)
+ return name.endswith(CYCLONEDX_JSON_SUFFIXES) or (pathlib.Path(name).name in CYCLONEDX_JSON_FILENAMES)
def is_cyclonedx_xml(name: str) -> bool:
- return name.endswith(CYCLONEDX_XML_SUFFIXES)
+ return name.endswith(CYCLONEDX_XML_SUFFIXES) or (pathlib.Path(name).name in CYCLONEDX_XML_FILENAMES)
def is_sbom_metadata(name: str) -> bool:
@@ -503,6 +520,11 @@ def sbom_candidates(name: str, suffixes: tuple[str, ...]) -> list[str]:
for stem in stems:
base = stem if (str(directory) == ".") else f"{directory}/{stem}"
candidates.extend(base + suffix for suffix in suffixes)
+ if stems and _SOURCE_CLASSIFIER.search(stems[0]):
+ if any(suffix.endswith(".json") for suffix in suffixes):
+ candidates.extend(str(path.with_name(filename)) for filename in CYCLONEDX_JSON_FILENAMES)
+ if any(suffix.endswith(".xml") for suffix in suffixes):
+ candidates.extend(str(path.with_name(filename)) for filename in CYCLONEDX_XML_FILENAMES)
return candidates
@@ -553,7 +575,10 @@ def _classifier_removed(name: str) -> str:
# must still be attached for this to work, as variant_pattern wants a trailing separator and
# the extension's dot provides it
pattern = r"[_-](?:" + "|".join(VARIANT_PATTERNS) + r")(?=" + extension_pattern() + r")"
- return re.sub(pattern, "", name, count=1)
+ declassified = re.sub(pattern, "", name, count=1)
+ if (declassified != name) or ((stem := artifact_stem(name)) is None):
+ return declassified
+ return _SOURCE_CLASSIFIER.sub("", stem, count=1) + name[len(stem) :]
if __name__ == "__main__":
diff --git a/atr/tasks/checks/paths.py b/atr/tasks/checks/paths.py
index 04d9d645..5e86258b 100644
--- a/atr/tasks/checks/paths.py
+++ b/atr/tasks/checks/paths.py
@@ -43,7 +43,7 @@ _DOC_TREE_MAX_FILES: Final = 512
# Release policy fields which this check relies on - used for result caching
INPUT_POLICY_KEYS: Final[list[str]] = ["binary_artifact_paths", "source_artifact_paths", "download_path_suffix"]
INPUT_EXTRA_ARGS: Final[list[str]] = ["is_podling", "all_files"]
-CHECK_VERSION: Final[str] = "9"
+CHECK_VERSION: Final[str] = "10"
async def check(args: checks.FunctionArguments) -> results.Results | None:
@@ -355,6 +355,9 @@ async def _check_path_process_single( # noqa: C901
ext_metadata = suffix
is_standalone_metadata = True
break
+ if (not ext_metadata) and analysis.is_cyclonedx(path.name):
+ ext_metadata = path.suffix
+ is_standalone_metadata = True
if (not ext_artifact) and (not ext_metadata) and analysis.is_sbom_metadata(path.name):
ext_metadata = path.suffix
@@ -459,7 +462,7 @@ def _is_bundled_doc(relative_path: safe.RelPath) -> bool:
search = re.search(analysis.extension_pattern(), path_str)
if search and (search.group("artifact") or search.group("metadata")):
return False
- return not any(path_str.endswith(suffix) for suffix in analysis.STANDALONE_METADATA_SUFFIXES)
+ return not analysis.is_cyclonedx(path_str)
async def _record(
diff --git a/tests/unit/test_classify.py b/tests/unit/test_classify.py
index 94f3a591..ac652f90 100644
--- a/tests/unit/test_classify.py
+++ b/tests/unit/test_classify.py
@@ -18,6 +18,8 @@
import json
+import pytest
+
import atr.classify as classify
import atr.models.safe as safe
@@ -179,15 +181,15 @@ async def test_sbom_classified_as_sbom():
async def test_sbom_content_marker_classifies_unsuffixed_json_as_sbom(tmp_path):
- path = safe.RelPath("bom.json")
- (tmp_path / "bom.json").write_text(json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6"}))
+ path = safe.RelPath("inventory.json")
+ (tmp_path / "inventory.json").write_text(json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6"}))
result = await classify.classify(path, base_path=safe.StatePath(tmp_path))
assert result == classify.FileType.SBOM
async def test_sbom_content_marker_classifies_unsuffixed_xml_as_sbom(tmp_path):
- path = safe.RelPath("bom.xml")
- (tmp_path / "bom.xml").write_text('<?xml version="1.0"?><bom xmlns="http://cyclonedx.org/schema/bom/1.6" />')
+ path = safe.RelPath("inventory.xml")
+ (tmp_path / "inventory.xml").write_text('<?xml version="1.0"?><bom xmlns="http://cyclonedx.org/schema/bom/1.6" />')
result = await classify.classify(path, base_path=safe.StatePath(tmp_path))
assert result == classify.FileType.SBOM
@@ -199,6 +201,14 @@ async def test_sbom_content_marker_ignores_unrelated_json(tmp_path):
assert result != classify.FileType.SBOM
+@pytest.mark.parametrize("name", ["sbom.json", "bom.xml", "sbom.json.asc", "sbom.json.sha512", "example-sbom.json"])
+async def test_sbom_names_classify_consistently(name: str) -> None:
+ path = safe.RelPath(f"sub/{name}")
+ expected = classify.FileType.METADATA if name.endswith((".asc", ".sha512")) else classify.FileType.SBOM
+ assert classify.classify_path(path.as_path()) == expected
+ assert await classify.classify(path) == expected
+
+
async def test_sbom_signature_classified_as_metadata():
path = safe.RelPath("apache-widget-1.0.tar.gz.cdx.json.asc")
assert (await classify.classify(path)) == classify.FileType.METADATA
diff --git a/tests/unit/test_documentation_tree_check.py b/tests/unit/test_documentation_tree_check.py
index 3179443c..37e36ddd 100644
--- a/tests/unit/test_documentation_tree_check.py
+++ b/tests/unit/test_documentation_tree_check.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
+import unittest.mock as mock
+
import pytest
import atr.models.safe as safe
@@ -56,8 +58,20 @@ def test_is_bundled_doc_excludes_artifacts_metadata_and_top_level():
assert paths._is_bundled_doc(safe.RelPath("binaries/de/app-1.0.tar.gz")) is False
assert paths._is_bundled_doc(safe.RelPath("binaries/de/app-1.0.tar.gz.asc")) is False
assert paths._is_bundled_doc(safe.RelPath("sub/sbom.cdx.json")) is False
+ assert paths._is_bundled_doc(safe.RelPath("sub/sbom.json")) is False
assert paths._is_bundled_doc(safe.RelPath("index.html")) is False
+@pytest.mark.parametrize("name", ["sbom.json", "sbom.json.asc", "sbom.json.sha512", "camel-k-2.11.0-sbom.json"])
+async def test_sbom_paths_accept_source_companions(monkeypatch, tmp_path, name: str) -> None:
+ recorder = _recorder(tmp_path)
+ monkeypatch.setattr(paths.user, "is_admin_async", mock.AsyncMock(return_value=False))
+ relative_paths = {"camel-k-sources-2.11.0.tar.gz", "sbom.json", name}
+ await paths._check_path_process_single(
+ "test", safe.StatePath(tmp_path), safe.RelPath(name), recorder, recorder, recorder, relative_paths, False
+ )
+ assert [status for status, _, _ in recorder.messages] == ["note"]
+
+
def _recorder(tmp_path) -> RecorderStub:
return RecorderStub(safe.StatePath(tmp_path), "atr.tasks.checks.paths.check_errors", "5")
diff --git a/tests/unit/test_sbom_candidates.py b/tests/unit/test_sbom_candidates.py
index 05c9570c..7ccff404 100644
--- a/tests/unit/test_sbom_candidates.py
+++ b/tests/unit/test_sbom_candidates.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
+import pytest
+
import atr.analysis as analysis
@@ -35,13 +37,22 @@ def test_sbom_candidates() -> None:
assert analysis.sbom_candidates("apache-example-1.0.jar", analysis.CYCLONEDX_JSON_SUFFIXES) == [
"apache-example-1.0.jar.cdx.json",
"apache-example-1.0.jar-cyclonedx.json",
+ "apache-example-1.0.jar.cyclonedx.json",
+ "apache-example-1.0.jar-sbom.json",
+ "apache-example-1.0.jar-bom.json",
"apache-example-1.0.cdx.json",
"apache-example-1.0-cyclonedx.json",
+ "apache-example-1.0.cyclonedx.json",
+ "apache-example-1.0-sbom.json",
+ "apache-example-1.0-bom.json",
]
# A name with no artifact extension has only itself to offer
assert analysis.sbom_candidates("KEYS", analysis.CYCLONEDX_JSON_SUFFIXES) == [
"KEYS.cdx.json",
"KEYS-cyclonedx.json",
+ "KEYS.cyclonedx.json",
+ "KEYS-sbom.json",
+ "KEYS-bom.json",
]
# The XML suffixes reach the stem too, which is what the cataloguer and announce pair on
assert "apache-example-1.0-cyclonedx.xml" in analysis.sbom_candidates(
@@ -49,6 +60,30 @@ def test_sbom_candidates() -> None:
)
+def test_sbom_candidates_generic_priority_and_directory() -> None:
+ candidates = analysis.sbom_candidates("source/camel-k-sources-2.11.0.tar.gz", analysis.SBOM_SUFFIXES)
+ assert candidates[-4:] == ["source/sbom.json", "source/bom.json", "source/sbom.xml", "source/bom.xml"]
+ assert "sbom.json" not in candidates
+ assert candidates.index("source/camel-k-2.11.0-sbom.json") < candidates.index("camel-k-2.11.0-sbom.json")
+ assert candidates.index("source/camel-k-2.11.0-sbom.json") < candidates.index("source/camel-k-2.11.0-bom.xml")
+ json_candidates = analysis.sbom_candidates("camel-k-sources-2.11.0.tar.gz", analysis.CYCLONEDX_JSON_SUFFIXES)
+ assert json_candidates[-2:] == ["sbom.json", "bom.json"]
+ assert not any(path.endswith(".xml") for path in json_candidates)
+
+
+@pytest.mark.parametrize(
+ "name",
+ [
+ "apache-source-tool-1.0.tar.gz",
+ "camel-k-client-2.11.0-linux-amd64.tar.gz",
+ "apache-example-1.0.tar.gz",
+ "apache-example-1.0-sources.tar.gz.asc",
+ ],
+)
+def test_sbom_candidates_generic_requires_source_classifier(name: str) -> None:
+ assert "sbom.json" not in analysis.sbom_candidates(name, analysis.SBOM_SUFFIXES)
+
+
def test_sbom_candidates_maven_classifier() -> None:
# The CycloneDX Maven plugin names its output off the version alone, dropping the artifact's
# own classifier, so a classified artifact still has to reach the declassified SBOM name
@@ -98,10 +133,28 @@ def test_sbom_pairs_artifact() -> None:
)
+@pytest.mark.parametrize(
+ ("artifact", "sbom"),
+ [
+ ("apache-ant-antunit-1.5.0-bin.tar.gz", "apache-ant-antunit-1.5.0-bin.tar.gz.cyclonedx.json"),
+ ("apache-camel-4.22.0-src.zip", "apache-camel-4.22.0-sbom.json"),
+ ("apache-camel-4.22.0.pom", "apache-camel-4.22.0-sbom.json"),
+ ("apache-ldap-api-2.1.7-src.zip", "apache-ldap-api-2.1.7-bom.xml"),
+ ("camel-kamelets-sources-4.8.5.tar.gz", "camel-kamelets-4.8.5-sbom.json"),
+ ("camel-k-sources-2.11.0.tar.gz", "sbom.json"),
+ ("apache-example-1.0-source-release.zip", "bom.xml"),
+ ("apache-example-src-1.0.0-beta-1.zip", "apache-example-1.0.0-beta-1-cyclonedx.json"),
+ ],
+)
+def test_sbom_pairs_artifact_conventions(artifact: str, sbom: str) -> None:
+ assert analysis.sbom_pairs_artifact(sbom, artifact, analysis.SBOM_SUFFIXES)
+
+
def test_classifier_removed() -> None:
# A classifier abutting the extension comes off, along with its leading separator
assert analysis._classifier_removed("apache-example-1.0-source-release.zip") == "apache-example-1.0.zip"
assert analysis._classifier_removed("apache-example-1.0-bin.tar.gz") == "apache-example-1.0.tar.gz"
+ assert analysis._classifier_removed("camel-kamelets-sources-4.8.5.tar.gz") == "camel-kamelets-4.8.5.tar.gz"
# A name with no classifier is returned untouched, so callers can tell nothing was stripped
assert analysis._classifier_removed("apache-example-1.0.jar") == "apache-example-1.0.jar"
# A variant word elsewhere in the name is not a classifier, so it stays put
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org