You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/09/07 16:10:08 UTC
(tooling-trusted-releases) branch sbp updated (b5a8b0df -> 9a2009d7)
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a change to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
discard b5a8b0df Fix some problems with SBOM license identifier recognition
add dfcc468c #1149 - Add logging framework to SBOM CLI
new 9a2009d7 Fix some problems with SBOM license identifier recognition
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b5a8b0df)
\
N -- N -- N refs/heads/sbp (9a2009d7)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
atr/sbom/cli.py | 41 +++++++++++++++++++++++++++++++++++------
1 file changed, 35 insertions(+), 6 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-trusted-releases) 01/01: Fix some problems with SBOM license identifier recognition
Posted by sb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 9a2009d723b395cd0550a2dee207eab75f7459ad
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Mon Sep 7 16:17:35 2026 +0100
Fix some problems with SBOM license identifier recognition
---
atr/docs/sbom-workflows.md | 2 ++
atr/get/sbom.py | 68 +++++++++++++++++++++++----------------
atr/sbom/constants/licenses.py | 5 +++
atr/sbom/licenses.py | 13 +++-----
atr/sbom/spdx.py | 19 +++++++++--
tests/unit/test_get_sbom_tally.py | 18 +++++++++--
tests/unit/test_sbom_licenses.py | 39 ++++++++++++++++++++++
7 files changed, 123 insertions(+), 41 deletions(-)
diff --git a/atr/docs/sbom-workflows.md b/atr/docs/sbom-workflows.md
index 4e9df63a..7545c028 100644
--- a/atr/docs/sbom-workflows.md
+++ b/atr/docs/sbom-workflows.md
@@ -100,6 +100,8 @@ These include:
* vulnerability analysis
* tool version analysis
+License analysis places each declared license in a category of the [ASF third party license policy](https://www.apache.org/legal/resolved.html), reading the license strings that the SBOM tool wrote into the file. Those strings can be wrong. A project can declare its license inaccurately in its build file, and an SBOM tool can map a license name to the wrong identifier. A license that ATR does not recognise is treated as Category X. Check anything unexpected against the upstream project. [...]
+
The scoring workflow is implemented through:
* `score_tool`
diff --git a/atr/get/sbom.py b/atr/get/sbom.py
index f7addf92..1f2d0ff3 100644
--- a/atr/get/sbom.py
+++ b/atr/get/sbom.py
@@ -269,11 +269,14 @@ def _choice_badges(choice: sbom.models.licenses.Choice) -> list[htm.Element]:
# A single licence, or an AND that all applies, shows one badge for the whole expression. An OR
# ATR resolved shows the chosen half in its category colour and the halves it set aside drained
# of colour, so a reader sees what was on offer without the alternatives reading as a verdict
+ name = choice.expression if (choice.chosen is None) else choice.chosen
+ if choice.any_unknown:
+ name = f"{name} (unrecognised license)"
if choice.chosen is None:
- return [_license_badge(choice.expression, choice.category)]
+ return [_license_badge(name, choice.category)]
return [
- _license_badge(choice.chosen, choice.category),
- *(_license_alternative_badge(name, category) for name, category in choice.alternatives),
+ _license_badge(name, choice.category),
+ *(_license_alternative_badge(other, category) for other, category in choice.alternatives),
]
@@ -303,6 +306,10 @@ def _license_category_style(category: sbom.models.licenses.Category) -> str:
return ".bg-danger"
+def _license_group_label(category: sbom.models.licenses.Category, any_unknown: bool) -> str:
+ return f"Category {category!s} (unrecognised license)" if any_unknown else f"Category {category!s}"
+
+
def _license_section(block: htm.Block, task: sql.Task | None) -> None:
block.h3["Licenses"]
# The licence issues come from the score, so without one there is nothing to list
@@ -310,6 +317,17 @@ def _license_section(block: htm.Block, task: sql.Task | None) -> None:
if task_result is None:
block.p[_score_unavailable(task)]
return
+ block.p[
+ "Categories follow the ",
+ htm.a(href="https://www.apache.org/legal/resolved.html")["ASF third party license policy"],
+ " and are read from the license strings that the SBOM tool wrote into this file. A project can declare"
+ " its license wrongly in its build file, and an SBOM tool can map a license name to the wrong"
+ " identifier. A license that ATR does not recognise is treated as Category X. Check anything unexpected"
+ " against the upstream project, and report a wrong declaration to that project, a wrong mapping to the"
+ " SBOM tool, and a correctly spelled license that ATR fails to place, or places wrongly, to ",
+ htm.a(href="https://github.com/apache/tooling-trusted-release/issues")["ATR"],
+ ".",
+ ]
warnings = []
errors = []
prev_licenses = None
@@ -340,13 +358,11 @@ def _license_table(
warning_rows = [
htm.tr[
htm.td[
- f"Category {category!s}"
- if (len(components) == 0)
- else htm.details[htm.summary[f"Category {category!s}"], htm.div[_detail_table(components)]]
+ label if (len(components) == 0) else htm.details[htm.summary[label], htm.div[_detail_table(components)]]
],
htm.td[f"{count!s} {f'({new!s} new, {updated!s} changed)' if (new or updated) else ''}"],
]
- for category, count, new, updated, components in _license_tally(items, prev)
+ for label, count, new, updated, components in _license_tally(items, prev)
]
block.table(".table.table-sm.table-bordered.table-striped")[
htm.thead[htm.tr[htm.th["License Category"], htm.th["Count"]]],
@@ -358,16 +374,17 @@ def _license_table(
def _license_tally(
items: list[sbom.models.licenses.Issue],
old_issues: list[sbom.models.licenses.Issue] | None,
-) -> list[tuple[sbom.models.licenses.Category, int, int | None, int | None, list[str | None]]]:
- counts: dict[sbom.models.licenses.Category, int] = {}
- components: dict[sbom.models.licenses.Category, list[str | None]] = {}
- new_counts: dict[sbom.models.licenses.Category, int] = {}
- updated_counts: dict[sbom.models.licenses.Category, int] = {}
+) -> list[tuple[str, int, int | None, int | None, list[str | None]]]:
+ counts: dict[tuple[sbom.models.licenses.Category, bool], int] = {}
+ components: dict[tuple[sbom.models.licenses.Category, bool], list[str | None]] = {}
+ new_counts: dict[tuple[sbom.models.licenses.Category, bool], int] = {}
+ updated_counts: dict[tuple[sbom.models.licenses.Category, bool], int] = {}
old_map = {lic.component_name: (lic.license_expression, lic.category) for lic in old_issues} if old_issues else None
for item in items:
- key = item.category
+ key = (item.category, item.any_unknown)
counts[key] = counts.get(key, 0) + 1
- name = str(item).capitalize()
+ text = str(item)
+ name = text[0].upper() + text[1:]
if old_map is not None:
if item.component_name not in old_map:
new_counts[key] = new_counts.get(key, 0) + 1
@@ -381,19 +398,16 @@ def _license_tally(
components[key] = [name]
else:
components[key].append(name)
- return sorted(
- [
- (
- category,
- count,
- new_counts.get(category, 0) if (old_issues is not None) else None,
- updated_counts.get(category, 0) if (old_issues is not None) else None,
- components.get(category, []),
- )
- for category, count in counts.items()
- ],
- key=lambda kv: kv[0].value,
- )
+ return [
+ (
+ _license_group_label(*key),
+ count,
+ new_counts.get(key, 0) if (old_issues is not None) else None,
+ updated_counts.get(key, 0) if (old_issues is not None) else None,
+ components.get(key, []),
+ )
+ for key, count in sorted(counts.items(), key=lambda kv: (kv[0][0].value, kv[0][1]))
+ ]
def _licenses_cell(choices: list[sbom.models.licenses.Choice]) -> htm.Element | str:
diff --git a/atr/sbom/constants/licenses.py b/atr/sbom/constants/licenses.py
index d580dcfb..fc5918ed 100644
--- a/atr/sbom/constants/licenses.py
+++ b/atr/sbom/constants/licenses.py
@@ -151,6 +151,11 @@ LICENSES: Final[dict[str, list[str]]] = {
],
}
+LICENSE_ALIASES: Final[dict[str, str]] = {
+ "EPLv2": "EPL-2.0",
+ "GPL-2.0-with-classpath-exception": "GPL-2.0-only WITH Classpath-exception-2.0",
+}
+
# The SPDX name for each categorised licence, so that a component declaring its licence by
# name instead of by identifier can still be categorised
LICENSE_NAMES: Final[dict[str, str]] = {
diff --git a/atr/sbom/licenses.py b/atr/sbom/licenses.py
index e2bacf68..ab0de6ea 100644
--- a/atr/sbom/licenses.py
+++ b/atr/sbom/licenses.py
@@ -98,9 +98,10 @@ def assess(license_expr: str, is_expression: bool) -> models.licenses.Choice:
# friendliest half decides and the rest are set aside as alternatives. Anything else - an AND,
# parentheses, or a licence standing on its own - all applies at once, so there the sternest
# half wins and there is no choice to offer
+ disjunction = False
if is_expression:
try:
- atoms = license_expression_atoms(license_expr)
+ atoms, disjunction = license_expression_atoms(license_expr)
except ValueError:
atoms = {license_expr}
else:
@@ -109,7 +110,7 @@ def assess(license_expr: str, is_expression: bool) -> models.licenses.Choice:
ordered = sorted(atoms)
categories = {atom: _atom_category(atom) for atom in ordered}
- if is_expression and _is_simple_disjunction(license_expr):
+ if disjunction:
chosen = min(ordered, key=lambda atom: _category_severity(categories[atom]))
return models.licenses.Choice(
expression=license_expr,
@@ -160,16 +161,12 @@ def _folded_atom(atom: str) -> str:
return (resolved if (resolved is not None) else atom).casefold()
-def _is_simple_disjunction(expr: str) -> bool:
- # SPDX writes its operators in capitals, so a plain "A OR B" is easy to spot. Parentheses or an
- # AND could hide a licence that all applies at once, so we relax only the unambiguous shape
- return (" OR " in expr) and (" AND " not in expr) and ("(" not in expr)
-
-
def _name_to_id() -> dict[str, str]:
index: dict[str, str] = {}
for license_id, name in constants.licenses.LICENSE_NAMES.items():
index[_normalised_name(name)] = license_id
+ for alias, license_id in constants.licenses.LICENSE_ALIASES.items():
+ index[_normalised_name(alias)] = license_id
for license_ids in constants.licenses.LICENSES.values():
for license_id in license_ids:
index.setdefault(_normalised_name(license_id), license_id)
diff --git a/atr/sbom/spdx.py b/atr/sbom/spdx.py
index 7f15eef6..ba7a6494 100644
--- a/atr/sbom/spdx.py
+++ b/atr/sbom/spdx.py
@@ -25,12 +25,18 @@ class LicenseExpressionParser:
self.items = items
self.text = text
self.position = 0
+ self.depth = 0
+ self.disjunctive = False
+ self.plain = True
- def parse(self) -> set[str]:
+ def enclosing(self, start: int) -> bool:
+ return (start == self.depth) and (self.position == (len(self.items) - self.depth))
+
+ def parse(self) -> tuple[set[str], bool]:
atoms, _ = self.parse_expression()
if self.position != len(self.items):
raise ValueError(self.text)
- return atoms
+ return atoms, (self.disjunctive and self.plain)
def parse_conjunction(self) -> tuple[set[str], bool]:
atoms, simple = self.parse_with()
@@ -38,6 +44,7 @@ class LicenseExpressionParser:
self.position += 1
atoms |= self.parse_with()[0]
simple = False
+ self.plain = False
return atoms, simple
def parse_expression(self) -> tuple[set[str], bool]:
@@ -46,6 +53,7 @@ class LicenseExpressionParser:
self.position += 1
atoms |= self.parse_conjunction()[0]
simple = False
+ self.disjunctive = True
return atoms, simple
def parse_primary(self, for_addition: bool) -> tuple[set[str], bool]:
@@ -53,11 +61,16 @@ class LicenseExpressionParser:
raise ValueError(self.text)
kind, value = self.items[self.position]
if kind == "LPAREN":
+ start = self.position
self.position += 1
+ self.depth += 1
atoms, _ = self.parse_expression()
if not self.peek("RPAREN"):
raise ValueError(self.text)
self.position += 1
+ self.depth -= 1
+ if not self.enclosing(start):
+ self.plain = False
return atoms, False
if (not for_addition) and (kind in {"ID", "LICREF", "DOCREF"}):
self.position += 1
@@ -86,7 +99,7 @@ class LicenseExpressionParser:
return (self.position < len(self.items)) and (self.items[self.position][0] == kind)
-def license_expression_atoms(expr: str) -> set[str]:
+def license_expression_atoms(expr: str) -> tuple[set[str], bool]:
pos = 0
tokens: list[tuple[str, str]] = []
for match in constants.spdx.TOKEN.finditer(expr):
diff --git a/tests/unit/test_get_sbom_tally.py b/tests/unit/test_get_sbom_tally.py
index 3e4d0038..29e49a14 100644
--- a/tests/unit/test_get_sbom_tally.py
+++ b/tests/unit/test_get_sbom_tally.py
@@ -29,8 +29,17 @@ def test_license_tally_counts_new_and_changed_within_each_category() -> None:
tally = {row[0]: row for row in sbom._license_tally(items, old)}
# Category, count, new, changed
- assert tally[models.licenses.Category.X][:4] == (models.licenses.Category.X, 1, 1, 0)
- assert tally[models.licenses.Category.B][:4] == (models.licenses.Category.B, 1, 0, 1)
+ assert tally["Category X"][:4] == ("Category X", 1, 1, 0)
+ assert tally["Category B"][:4] == ("Category B", 1, 0, 1)
+
+
+def test_license_tally_keeps_the_declared_license_string_as_written() -> None:
+ items = [_issue("fresh", "EPLv2", models.licenses.Category.X, any_unknown=True)]
+
+ (row,) = sbom._license_tally(items, None)
+
+ assert row[0] == "Category X (unrecognised license)"
+ assert row[4] == ["Component fresh@1.0 declares license EPLv2"]
def test_license_tally_omits_comparison_counts_without_a_previous_release() -> None:
@@ -42,10 +51,13 @@ def test_license_tally_omits_comparison_counts_without_a_previous_release() -> N
assert row[3] is None
-def _issue(name: str, expression: str, category: models.licenses.Category) -> models.licenses.Issue:
+def _issue(
+ name: str, expression: str, category: models.licenses.Category, any_unknown: bool = False
+) -> models.licenses.Issue:
return models.licenses.Issue(
component_name=name,
component_version="1.0",
license_expression=expression,
category=category,
+ any_unknown=any_unknown,
)
diff --git a/tests/unit/test_sbom_licenses.py b/tests/unit/test_sbom_licenses.py
index ab890f00..f9758a25 100644
--- a/tests/unit/test_sbom_licenses.py
+++ b/tests/unit/test_sbom_licenses.py
@@ -30,6 +30,12 @@ def test_expression() -> None:
assert licenses.expression(cdx_license.DisjunctiveLicense(name="Some bespoke licence")) == "Some bespoke licence"
+def test_assess_chooses_within_a_disjunction_whose_operand_carries_an_exception() -> None:
+ reading = licenses.assess("MIT OR GPL-2.0-only WITH Classpath-exception-2.0", is_expression=True)
+
+ assert (reading.category, reading.chosen) == (models.licenses.Category.A, "MIT")
+
+
def test_assess_picks_the_friendliest_half_of_a_disjunction_and_sets_the_rest_aside() -> None:
reading = licenses.assess("Apache-2.0 OR GPL-3.0-only", is_expression=True)
@@ -47,6 +53,23 @@ def test_assess_offers_no_choice_for_a_single_licence_or_a_conjunction() -> None
assert (conjunction.chosen, conjunction.category) == (None, models.licenses.Category.X)
+def test_assess_offers_no_choice_where_a_conjunction_sits_inside_a_disjunction() -> None:
+ nested = licenses.assess("Apache-2.0 OR (MIT AND GPL-3.0-only)", is_expression=True)
+ leading = licenses.assess("(Apache-2.0 AND MIT) OR GPL-3.0-only", is_expression=True)
+
+ assert (nested.chosen, nested.category) == (None, models.licenses.Category.X)
+ assert (leading.chosen, leading.category) == (None, models.licenses.Category.X)
+
+
+def test_assess_unwraps_outer_parentheses_around_a_disjunction() -> None:
+ maven = licenses.assess("(CDDL-1.0 OR GPL-2.0-with-classpath-exception)", is_expression=True)
+ doubled = licenses.assess("((MIT OR GPL-3.0-only))", is_expression=True)
+
+ assert (maven.category, maven.chosen) == (models.licenses.Category.B, "CDDL-1.0")
+ assert maven.alternatives == [("GPL-2.0-with-classpath-exception", models.licenses.Category.X)]
+ assert (doubled.category, doubled.chosen) == (models.licenses.Category.A, "MIT")
+
+
def test_check() -> None:
good, warnings, errors = licenses.check(
sboms.with_components(
@@ -114,6 +137,22 @@ def test_check_categorises_a_public_domain_declaration_as_category_a() -> None:
assert errors == []
+def test_check_categorises_an_aliased_spelling() -> None:
+ _good, warnings, errors = licenses.check(
+ sboms.with_components(
+ {"type": "library", "name": "compact", "licenses": [{"license": {"name": "EPLv2"}}]},
+ {"type": "library", "name": "deprecated", "licenses": [{"expression": "GPL-2.0-with-classpath-exception"}]},
+ ),
+ )
+
+ assert [(issue.component_name, issue.category, issue.any_unknown) for issue in warnings] == [
+ ("compact", models.licenses.Category.B, False),
+ ]
+ assert [(issue.component_name, issue.category, issue.any_unknown) for issue in errors] == [
+ ("deprecated", models.licenses.Category.X, False),
+ ]
+
+
def test_check_a_disjunction_settles_on_the_friendliest_category() -> None:
# A component offered under "A OR B" may be taken under whichever half is easier to live with
good, warnings, errors = licenses.check(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org