You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/08/24 22:40:16 UTC

(tooling-trusted-releases) branch sbp updated (60154485 -> 5965924d)

This is an automated email from the ASF dual-hosted git repository.

sbp pushed a change to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git


    omit 60154485 Resolve ASF UIDs consistently in OpenPGP related code
     new 5965924d Resolve ASF UIDs consistently in OpenPGP related code

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (60154485)
            \
             N -- N -- N   refs/heads/sbp (5965924d)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 scripts/keys_backfill.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org


(tooling-trusted-releases) 01/01: Resolve ASF UIDs consistently in OpenPGP related code

Posted by sb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git

commit 5965924d40c96f9b258c8566b7b2f3e4016eddd9
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Mon Aug 24 22:44:18 2026 +0100

    Resolve ASF UIDs consistently in OpenPGP related code
---
 atr/storage/writers/keys.py      | 54 +++++++++++++++++++++++++++++++++++++++-
 scripts/keys_backfill.py         | 53 ++++++++++++++++++---------------------
 tests/unit/test_keys_backfill.py | 12 ---------
 tests/unit/test_keys_writer.py   | 49 ++++++++++++++++++++++++++++++++++++
 4 files changed, 126 insertions(+), 42 deletions(-)

diff --git a/atr/storage/writers/keys.py b/atr/storage/writers/keys.py
index 068b3a38..c26398cd 100644
--- a/atr/storage/writers/keys.py
+++ b/atr/storage/writers/keys.py
@@ -194,6 +194,29 @@ def _signing_key_rows(certificate_fingerprint: str, block: str | bytes) -> list[
     ]
 
 
+def _resolved_apache_uid(
+    uids: list[str], current: str | None, users: frozenset[str], lookup: cache.EmailUidLookup
+) -> tuple[str | None, str]:
+    local_parts = []
+    emails = []
+    for uid in uids:
+        email = util.email_from_uid(uid)
+        if not email:
+            continue
+        if email.endswith("@apache.org"):
+            local_parts.append(email.removesuffix("@apache.org"))
+        emails.append(email)
+    mapped = [value for value in (lookup.get(email) for email in emails) if value]
+    if (current is not None) and ((current in local_parts) or (current in mapped)):
+        return current, "stable"
+    for part in local_parts:
+        if part in users:
+            return part, "apache-uid"
+    for value in mapped:
+        return value, "ldap-map"
+    return None, "none"
+
+
 def _source_string(source: datatypes.KeySource, committee_key: str | None = None) -> str:
     match source:
         case datatypes.KeySource.API | datatypes.KeySource.WEB:
@@ -272,7 +295,9 @@ class FoundationCommitter(GeneralPublic):
         self.__asf_uid = asf_uid
 
         # Specific to this module
+        self.__email_lookup_cache: cache.EmailUidLookup | None = None
         self.__key_block_models_cache = {}
+        self.__users_cache: frozenset[str] | None = None
 
     async def delete_key(self, fingerprint: str, source: datatypes.KeySource) -> outcome.Outcome[datatypes.KeyDeletion]:
         source_string = _source_string(source)
@@ -414,7 +439,18 @@ class FoundationCommitter(GeneralPublic):
         changed = result != previous
         if changed and ((len(block) > _MAX_CERTIFICATE_BYTES) or (len(result) > _MAX_CERTIFICATE_PLACEMENTS)):
             raise ValueError(f"Key {fingerprint} would exceed the size admitted for a certificate")
-        apache_uid = key.key_model.apache_uid or (existing.apache_uid if (existing is not None) else None)
+        apache_uid = existing.apache_uid if (existing is not None) else None
+        if changed or (existing is None):
+            users = await self.__user_uids()
+            resolved, _branch = _resolved_apache_uid(pgp.user_id_texts(block), apache_uid, users, self.__email_lookup())
+            candidate = key.key_model.apache_uid
+            if existing is not None:
+                apache_uid = resolved or existing.apache_uid
+            elif candidate and ((candidate in users) or (candidate == self.__asf_uid)):
+                apache_uid = resolved or candidate
+            else:
+                apache_uid = resolved
+        key.key_model.apache_uid = apache_uid
         head = self.__head_model(fingerprint, block, apache_uid)
         if changed:
             raw = None if (source.partition(":")[0] in _POINTER_SOURCE_SCHEMES) else key.input
@@ -668,6 +704,11 @@ class FoundationCommitter(GeneralPublic):
     ) -> tuple[outcome.Outcome[datatypes.Key], dict[str, outcome.Outcome[datatypes.KeysPublish]]]:
         await self.__data.begin_immediate()
         status = await self._revise_certificate(key, source)
+        if (key.key_model.apache_uid is None) and (status != datatypes.KeyStatus.PARSED):
+            await self.__data.rollback()
+            raise datatypes.UnknownApacheUidError(
+                "OpenPGP key could not be associated with an ASF UID. Import it through a KEYS file instead."
+            )
         if status == datatypes.KeyStatus.INSERTED:
             await self.__signature_hints_consume([key])
         await self.__data.commit()
@@ -687,6 +728,11 @@ class FoundationCommitter(GeneralPublic):
         log.info(f"Stored key {fingerprint}")
         return outcome.Result(datatypes.Key(status=status, key_model=key.key_model)), publications
 
+    def __email_lookup(self) -> cache.EmailUidLookup:
+        if self.__email_lookup_cache is None:
+            self.__email_lookup_cache = cache.email_uid_view()
+        return self.__email_lookup_cache
+
     async def __ensure_one(
         self, key_file_text: str, source: str, associate: bool = True
     ) -> tuple[outcome.Outcome[datatypes.Key], dict[str, outcome.Outcome[datatypes.KeysPublish]]]:
@@ -856,6 +902,12 @@ and was published by the committee.\
                 return ldap_data[email]
         return None
 
+    async def __user_uids(self) -> frozenset[str]:
+        if self.__users_cache is None:
+            rows = await self.__data.execute(sqlmodel.select(sql.validate_instrumented_attribute(sql.User.asfuid)))
+            self.__users_cache = frozenset(rows.scalars().all())
+        return self.__users_cache
+
 
 class CommitteeParticipant(FoundationCommitter):
     def __init__(
diff --git a/scripts/keys_backfill.py b/scripts/keys_backfill.py
index e3e14f8e..87e48fd2 100644
--- a/scripts/keys_backfill.py
+++ b/scripts/keys_backfill.py
@@ -203,6 +203,12 @@ def _armored_blocks(text: str) -> list[str]:
     return found
 
 
+def _automated_uid(uid: str) -> bool:
+    if not any(label in uid for label in util.AUTOMATED_RELEASE_SIGNING_LABELS):
+        return False
+    return ("private@" in uid) and (".apache.org" in uid)
+
+
 def _backfill(
     events: list[Event],
     scope: frozenset[str],
@@ -617,29 +623,6 @@ def _resolve(path: str, committees: frozenset[str], projects: dict[str, str]) ->
     return None, "none"
 
 
-def _resolve_uid(
-    uids: tuple[str, ...], current: str | None, users: frozenset[str], lookup: cache.EmailUidLookup
-) -> tuple[str | None, str]:
-    local_parts = []
-    emails = []
-    for uid in uids:
-        email = util.email_from_uid(uid)
-        if not email:
-            continue
-        if email.endswith("@apache.org"):
-            local_parts.append(email.removesuffix("@apache.org"))
-        emails.append(email)
-    mapped = [value for value in (lookup.get(email) for email in emails) if value]
-    if (current is not None) and ((current in local_parts) or (current in mapped)):
-        return current, "stable"
-    for part in local_parts:
-        if part in users:
-            return part, "apache-uid"
-    for value in mapped:
-        return value, "ldap-map"
-    return None, "none"
-
-
 async def _run(args: argparse.Namespace) -> None:
     await db.init_database_for_worker()
     blocks: Blocks = {}
@@ -695,7 +678,7 @@ async def _run(args: argparse.Namespace) -> None:
     lookup = cache.email_uid_view()
     changes = _sweep(certificates, heads, users, lookup)
     inserted = {
-        fingerprint: _resolve_uid(heads[fingerprint].uids, None, users, lookup)[0]
+        fingerprint: keys_writer._resolved_apache_uid(list(heads[fingerprint].uids), None, users, lookup)[0]
         for fingerprint in sorted(frozenset(listed.coverage) - stored_fingerprints - frozenset(_DELETED_FINGERPRINTS))
         if fingerprint in heads
     }
@@ -704,6 +687,11 @@ async def _run(args: argparse.Namespace) -> None:
         for committee, fingerprint in listed.links
         if (fingerprint in scope) and ((committee, fingerprint) not in existing_links)
     )
+    automated_links = frozenset(
+        committee
+        for committee, fingerprint in links_added
+        if (fingerprint in heads) and heads[fingerprint].uids and _automated_uid(heads[fingerprint].uids[0])
+    )
     problems += [
         f"inserted certificate {fingerprint} matches a signature hint; import it through the writer instead"
         for fingerprint in _hint_matches(inserted, heads, hints)
@@ -714,6 +702,8 @@ async def _run(args: argparse.Namespace) -> None:
         f" unresolved paths {len(listed.unresolved)}; rejected blocks {len(listed.rejected)}"
     )
     print(f"switchover digest {_switchover_digest(inserted, links_added, listed)}")
+    if automated_links:
+        print(f"automation additions via new links: {sorted(automated_links)}")
     if problems:
         sys.exit(1)
     if not args.apply:
@@ -729,7 +719,7 @@ async def _run(args: argparse.Namespace) -> None:
     args.manifest.write_text('{"incomplete": true}\n')
     listener = _setup_audit_logging()
     try:
-        await _write(backfill, heads, automated_before, inserted, links_added)
+        await _write(backfill, heads, automated_before, automated_links, inserted, links_added)
         await _write_sweep(changes)
     finally:
         listener.stop()
@@ -883,7 +873,7 @@ def _sweep(
         head = heads.get(fingerprint)
         if head is None:
             continue
-        value, branch = _resolve_uid(head.uids, certificate.apache_uid, users, lookup)
+        value, branch = keys_writer._resolved_apache_uid(list(head.uids), certificate.apache_uid, users, lookup)
         if value != certificate.apache_uid:
             changes.append(UidChange(fingerprint, certificate.apache_uid, value, branch))
     return changes
@@ -983,6 +973,7 @@ async def _write(
     backfill: Backfill,
     heads: dict[str, Head],
     automated_before: frozenset[str],
+    automated_links: frozenset[str],
     inserted: dict[str, str | None],
     links_added: list[tuple[str, str]],
 ) -> None:
@@ -1041,9 +1032,13 @@ async def _write(
         await data.ns_text_set("openpgp-log", f"archive:{_ARCHIVE_CAPTURE_DATE}", _ARCHIVE_CAPTURE_SHA256, commit=False)
         await _verify_database(data, frozenset(backfill.final), heads)
         automated_after = await interaction.automated_release_signing_committees(data)
-        if automated_after != automated_before:
-            changed = sorted(automated_before ^ automated_after)
-            raise RuntimeError(f"the automated release signing set changed: {changed}")
+        removed = automated_before - automated_after
+        unexpected = automated_after - automated_before - automated_links
+        if removed or unexpected:
+            raise RuntimeError(
+                f"the automated release signing set changed:"
+                f" removed {sorted(removed)}, unexplained {sorted(unexpected)}"
+            )
         await data.commit()
     for fingerprint in _DELETED_FINGERPRINTS:
         storage.audit(action="key_hard_delete", fingerprint=fingerprint, phase="backfill")
diff --git a/tests/unit/test_keys_backfill.py b/tests/unit/test_keys_backfill.py
index b6a3d86a..e2b25fee 100644
--- a/tests/unit/test_keys_backfill.py
+++ b/tests/unit/test_keys_backfill.py
@@ -20,7 +20,6 @@ import importlib.util
 import pathlib
 import sys
 
-import atr.cache as cache
 import atr.models.sql as sql
 import atr.pgp as pgp
 import tests.unit.pgp_fixtures as pgp_fixtures
@@ -84,17 +83,6 @@ def test_backfill_reports_missing_chains_and_actors() -> None:
     assert any("no chain" in problem for problem in backfill.problems)
 
 
-def test_resolve_uid_prefers_stability_then_existence_then_map() -> None:
-    lookup = cache.EmailUidLookup.from_plain({"person@example.com": "mapped", "alias@apache.org": "aliased"})
-    users = frozenset({"real"})
-    uids = ("A <gh...@apache.org>", "B <re...@apache.org>", "C <pe...@example.com>")
-    assert keys_backfill._resolve_uid(uids, "ghost", users, lookup) == ("ghost", "stable")
-    assert keys_backfill._resolve_uid(uids, None, users, lookup) == ("real", "apache-uid")
-    assert keys_backfill._resolve_uid(("C <pe...@example.com>",), None, users, lookup) == ("mapped", "ldap-map")
-    assert keys_backfill._resolve_uid(("E <al...@apache.org>",), None, users, lookup) == ("aliased", "ldap-map")
-    assert keys_backfill._resolve_uid(("D <ot...@example.com>",), "gone", users, lookup) == (None, "none")
-
-
 def test_resolve_follows_the_rehoming_rule() -> None:
     committees = frozenset({"accumulo", "activemq", "oltu"})
     projects = {"accumulo": "accumulo", "activemq-apollo": "activemq", "amber": "oltu"}
diff --git a/tests/unit/test_keys_writer.py b/tests/unit/test_keys_writer.py
index f8de0a24..60153134 100644
--- a/tests/unit/test_keys_writer.py
+++ b/tests/unit/test_keys_writer.py
@@ -25,6 +25,7 @@ import sqlalchemy
 import sqlalchemy.ext.asyncio
 import sqlmodel
 
+import atr.cache as cache
 import atr.db as db
 import atr.log as log
 import atr.models.sql as sql
@@ -968,3 +969,51 @@ async def test_unchanged_observation_of_an_over_limit_head_is_not_refused(sqlite
     result, _publications = await writer._FoundationCommitter__database_add_model(again, "web:req-2")
 
     assert result.result_or_raise().status == datatypes.KeyStatus.PARSED
+
+
+def test_resolved_apache_uid_prefers_stability_then_existence_then_map():
+    lookup = cache.EmailUidLookup.from_plain({"person@example.com": "mapped", "alias@apache.org": "aliased"})
+    users = frozenset({"real"})
+    uids = ["A <gh...@apache.org>", "B <re...@apache.org>", "C <pe...@example.com>"]
+    assert keys_writer._resolved_apache_uid(uids, "ghost", users, lookup) == ("ghost", "stable")
+    assert keys_writer._resolved_apache_uid(uids, None, users, lookup) == ("real", "apache-uid")
+    assert keys_writer._resolved_apache_uid(["C <pe...@example.com>"], None, users, lookup) == ("mapped", "ldap-map")
+    assert keys_writer._resolved_apache_uid(["E <al...@apache.org>"], None, users, lookup) == ("aliased", "ldap-map")
+    assert keys_writer._resolved_apache_uid(["D <ot...@example.com>"], "gone", users, lookup) == (None, "none")
+
+
+@pytest.mark.asyncio
+async def test_revise_prefers_the_existing_apache_uid_over_the_upload_candidate(sqlite_data):
+    full = pgp_fixtures.REVOKED_SUBKEY_PUBLIC_KEY_ASC
+    fingerprint = pgp_fixtures.REVOKED_SUBKEY_PRIMARY_FINGERPRINT
+    stripped = pgp_fixtures.block_without_signature_type(full, 0x28)
+    await _seed_committee_key(sqlite_data, "alpha", fingerprint, stripped)
+    writer, _write, _write_as = _make_foundation_committer_with_audit(sqlite_data)
+    key = _parsed_key(full)
+    key.key_model.apache_uid = "other"
+
+    result, _publications = await writer._FoundationCommitter__database_add_model(key, "web:req-2")
+
+    outcome_key = result.result_or_raise()
+    assert outcome_key.status == datatypes.KeyStatus.REFRESHED
+    assert outcome_key.key_model.apache_uid == "alice"
+    stored = await sqlite_data.signing_certificate(fingerprint=fingerprint).get()
+    assert stored is not None
+    assert stored.apache_uid == "alice"
+
+
+@pytest.mark.asyncio
+async def test_single_add_refuses_an_unvetted_candidate_for_a_new_certificate(sqlite_data):
+    writer, _write, _write_as = _make_foundation_committer_with_audit(sqlite_data)
+    key = _parsed_key(_ALPHA_BLOCK)
+    key.key_model.apache_uid = "ghost"
+
+    with pytest.raises(datatypes.UnknownApacheUidError):
+        await writer._FoundationCommitter__database_add_model(key, "web:req-1")
+
+    assert (await sqlite_data.signing_certificate(fingerprint=_ALPHA_FINGERPRINT).get()) is None
+    assert await sqlite_data.key_attestable(fingerprint=_ALPHA_FINGERPRINT).all() == []
+    retry = _parsed_key(_ALPHA_BLOCK)
+    retry.key_model.apache_uid = "alice"
+    result, _publications = await writer._FoundationCommitter__database_add_model(retry, "web:req-2")
+    assert result.result_or_raise().status == datatypes.KeyStatus.INSERTED


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org