You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/04/20 15:43:31 UTC
(tooling-trusted-releases) branch sbp updated: Add release file generation provenance to the database
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/sbp by this push:
new f2fc09ab Add release file generation provenance to the database
f2fc09ab is described below
commit f2fc09abd407556da3e7b064d6f3e76f6c6ee7e5
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Mon Apr 20 16:43:08 2026 +0100
Add release file generation provenance to the database
---
atr/attestable.py | 22 +++++-
atr/models/sql.py | 5 +-
atr/storage/writers/revision.py | 16 +++-
migrations/versions/0068_2026.04.20_24072a50.py | 29 ++++++++
tests/unit/test_create_revision.py | 99 +++++++++++++++++++++++++
tests/unit/test_release_file_state.py | 70 +++++++++++++++++
6 files changed, 237 insertions(+), 4 deletions(-)
diff --git a/atr/attestable.py b/atr/attestable.py
index a7d080fb..79f9932a 100644
--- a/atr/attestable.py
+++ b/atr/attestable.py
@@ -102,22 +102,40 @@ def compute_file_state_rows(
path_to_hash: dict[safe.RelPath, str],
classifications: dict[safe.RelPath, str],
previous: models.Attestable | None,
+ effective_path_provenance: dict[str, models.ProvenanceV2] | None = None,
) -> list[sql.ReleaseFileState]:
+ # This function is called when committing a new revision
+ # It's called after writing attestable data to attestable files
+ # Now we add some of that same data to ReleaseFileState rows
+ # We then return those rows for actually adding to the database
prev_hashes: dict[str, str] = {}
prev_classifications: dict[str, str] = {}
+ prev_provenance: dict[str, models.ProvenanceV2 | None] = {}
if previous is not None:
prev_hashes = path_hashes(previous)
if isinstance(previous, models.AttestableV2):
prev_classifications = {path_key: entry.classification for path_key, entry in previous.paths.items()}
+ prev_provenance = {path_key: entry.provenance for path_key, entry in previous.paths.items()}
+ effective = effective_path_provenance or {}
rows: list[sql.ReleaseFileState] = []
+ # ReleaseFileState only reflects attestable path state, i.e. omitting hashes and policy
+ # Also, we only record new rows when the metadata changes
+ # But in attestable files, we record the data always, for each revision
for path_key in sorted(path_to_hash, key=str):
path_str = str(path_key)
content_hash = path_to_hash[path_key]
classification = classifications[path_key]
+ provenance_entry = effective.get(path_str)
+ provenance_payload = provenance_entry.model_dump(mode="json") if (provenance_entry is not None) else None
+ prev_provenance_entry = prev_provenance.get(path_str)
# If all prior metadata properties are the same, we skip recording an event
- if (prev_hashes.get(path_str) == content_hash) and (prev_classifications.get(path_str) == classification):
+ if (
+ (prev_hashes.get(path_str) == content_hash)
+ and (prev_classifications.get(path_str) == classification)
+ and (prev_provenance_entry == provenance_entry)
+ ):
continue
rows.append(
sql.ReleaseFileState(
@@ -127,6 +145,7 @@ def compute_file_state_rows(
present=True,
content_hash=content_hash,
classification=classification,
+ provenance=provenance_payload,
)
)
str_keys = {str(k) for k in path_to_hash}
@@ -140,6 +159,7 @@ def compute_file_state_rows(
present=False,
content_hash=None,
classification=None,
+ provenance=None,
)
)
diff --git a/atr/models/sql.py b/atr/models/sql.py
index c1d8405e..7ba402fa 100644
--- a/atr/models/sql.py
+++ b/atr/models/sql.py
@@ -1363,6 +1363,9 @@ class ReleaseFileState(sqlmodel.SQLModel, table=True):
present: bool = sqlmodel.Field(**example(True))
content_hash: str | None = sqlmodel.Field(default=None, **example("blake3:7f83b1657ff1fc..."))
classification: str | None = sqlmodel.Field(default=None, **example("source"))
+ provenance: dict[str, Any] | None = sqlmodel.Field(
+ default=None, sa_column=sqlalchemy.Column(SafeJSON, nullable=True)
+ )
__table_args__ = (
sqlalchemy.ForeignKeyConstraint(
@@ -1375,7 +1378,7 @@ class ReleaseFileState(sqlmodel.SQLModel, table=True):
(
(present = 1 AND content_hash IS NOT NULL AND classification IS NOT NULL)
OR
- (present = 0 AND content_hash IS NULL AND classification IS NULL)
+ (present = 0 AND content_hash IS NULL AND classification IS NULL AND provenance IS NULL)
)
""",
name="valid_release_file_state",
diff --git a/atr/storage/writers/revision.py b/atr/storage/writers/revision.py
index 730dafad..f370ac8f 100644
--- a/atr/storage/writers/revision.py
+++ b/atr/storage/writers/revision.py
@@ -90,6 +90,7 @@ async def finalise_revision(
temp_dir_path: pathlib.Path,
version_key: safe.VersionKey,
was_quarantined: bool = False,
+ path_provenance: dict[safe.RelPath, atr.models.attestable.ProvenanceV2] | None = None,
) -> sql.Revision:
try:
previous_attestable, merge_base_revision_key, _, merged_release = await _lock_and_merge(
@@ -126,6 +127,7 @@ async def finalise_revision(
temp_dir=temp_dir,
version_key=version_key,
was_quarantined=was_quarantined,
+ path_provenance=path_provenance,
)
@@ -144,6 +146,7 @@ async def _commit_new_revision(
temp_dir: str,
version_key: safe.VersionKey,
was_quarantined: bool = False,
+ path_provenance: dict[safe.RelPath, atr.models.attestable.ProvenanceV2] | None = None,
) -> sql.Revision:
try:
# This is the only place where models.Revision is constructed
@@ -198,6 +201,7 @@ async def _commit_new_revision(
classifications = await attestable.compute_classifications(
path_to_hash, policy_dict, new_revision_dir, archives_base
)
+ effective_provenance = attestable.effective_path_provenance(path_provenance, path_to_hash, previous_attestable)
await attestable.write_files_data(
project_key,
@@ -210,6 +214,7 @@ async def _commit_new_revision(
path_to_size,
new_revision_dir,
classifications=classifications,
+ effective_path_provenance=effective_provenance,
)
if attestable.can_write_file_state_rows(previous_attestable, new_revision.parent_key):
@@ -219,6 +224,7 @@ async def _commit_new_revision(
path_to_hash,
classifications,
previous_attestable,
+ effective_path_provenance=effective_provenance,
):
data.add(row)
@@ -381,7 +387,11 @@ class CommitteeParticipant(FoundationCommitter):
description: str | None = None,
set_local_cache: bool = False,
reset_to_global_cache: bool = False,
- modify: Callable[[safe.StatePath, sql.Revision | None], Awaitable[None]] | None = None,
+ modify: Callable[
+ [safe.StatePath, sql.Revision | None],
+ Awaitable[dict[safe.RelPath, atr.models.attestable.ProvenanceV2] | None],
+ ]
+ | None = None,
clone_from: safe.RevisionNumber | None = None,
) -> sql.Revision | sql.Quarantined:
"""Create a new revision, quarantining archives that require validation."""
@@ -414,6 +424,7 @@ class CommitteeParticipant(FoundationCommitter):
prefix_token = secrets.token_hex(16)
temp_dir: str = await asyncio.to_thread(tempfile.mkdtemp, prefix=prefix_token + "-", dir=paths.get_tmp_dir())
temp_dir_path = safe.StatePath(pathlib.Path(temp_dir), paths.get_tmp_dir().path)
+ path_provenance: dict[safe.RelPath, atr.models.attestable.ProvenanceV2] | None = None
try:
# The directory was created by mkdtemp, but it's empty
@@ -422,7 +433,7 @@ class CommitteeParticipant(FoundationCommitter):
await util.create_hard_link_clone(old_release_dir, temp_dir_path, do_not_create_dest_dir=True)
# The directory is either empty or its files are hard linked to the previous revision
if modify is not None:
- await modify(temp_dir_path, old_revision)
+ path_provenance = await modify(temp_dir_path, old_revision)
except types.FailedError:
await aioshutil.rmtree(temp_dir)
raise
@@ -535,6 +546,7 @@ class CommitteeParticipant(FoundationCommitter):
release_key=release_key,
temp_dir=temp_dir,
version_key=version_key,
+ path_provenance=path_provenance,
)
async def _quarantine_archives(
diff --git a/migrations/versions/0068_2026.04.20_24072a50.py b/migrations/versions/0068_2026.04.20_24072a50.py
new file mode 100644
index 00000000..b9048abe
--- /dev/null
+++ b/migrations/versions/0068_2026.04.20_24072a50.py
@@ -0,0 +1,29 @@
+"""Add release file generation provenance
+
+Revision ID: 0068_2026.04.20_24072a50
+Revises: 0067_2026.04.14_bbfd41d3
+Create Date: 2026-04-20 15:39:52.177891+00:00
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+import atr.models.sql
+
+# Revision identifiers, used by Alembic
+revision: str = "0068_2026.04.20_24072a50"
+down_revision: str | None = "0067_2026.04.14_bbfd41d3"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ with op.batch_alter_table("releasefilestate", schema=None) as batch_op:
+ batch_op.add_column(sa.Column("provenance", atr.models.sql.SafeJSON(), nullable=True))
+
+
+def downgrade() -> None:
+ with op.batch_alter_table("releasefilestate", schema=None) as batch_op:
+ batch_op.drop_column("provenance")
diff --git a/tests/unit/test_create_revision.py b/tests/unit/test_create_revision.py
index 864111dd..f60c4c99 100644
--- a/tests/unit/test_create_revision.py
+++ b/tests/unit/test_create_revision.py
@@ -310,6 +310,105 @@ async def test_modify_failed_error_propagates_and_cleans_up(tmp_path: pathlib.Pa
assert not os.listdir(tmp_path)
+@pytest.mark.asyncio
+async def test_modify_path_provenance_flows_into_write_files_data(tmp_path: pathlib.Path):
+ temp_dir = safe.StatePath(tmp_path)
+ release = mock.MagicMock()
+ release.phase = sql.ReleasePhase.RELEASE_CANDIDATE_DRAFT
+ release.project = mock.MagicMock()
+ release.project.release_policy = None
+ release.release_policy = None
+
+ import contextlib
+
+ import atr.models.attestable as attestable_models
+
+ provenance = attestable_models.ProvenanceV2(
+ generator=attestable_models.GeneratorV2.SHA512_FROM_SIGNATURE,
+ metadata={"initiated_by": "alice", "source_paths": ["example.tar.gz"]},
+ )
+ provenance_map: dict[safe.RelPath, attestable_models.ProvenanceV2] = {
+ safe.RelPath("example.tar.gz.sha512"): provenance,
+ }
+
+ async def modify(
+ path: safe.StatePath, old_rev: object
+ ) -> dict[safe.RelPath, attestable_models.ProvenanceV2] | None:
+ return provenance_map
+
+ mock_session = _mock_db_session(release)
+ participant = _make_participant()
+ safe_data = MockSafeData(parent_key=None, new_number="00001")
+ write_files_data_mock = mock.AsyncMock()
+ compute_file_state_rows_mock = mock.MagicMock(return_value=[])
+
+ patches = [
+ mock.patch.object(revision.aiofiles.os, "makedirs", new_callable=mock.AsyncMock),
+ mock.patch.object(revision.aiofiles.os, "rename", new_callable=mock.AsyncMock),
+ mock.patch.object(revision.attestable, "load", new_callable=mock.AsyncMock, return_value=None),
+ mock.patch.object(
+ revision.attestable,
+ "paths_to_hashes_and_sizes",
+ new_callable=mock.AsyncMock,
+ return_value=(
+ {
+ safe.RelPath("example.tar.gz"): "blake3:src",
+ safe.RelPath("example.tar.gz.sha512"): "blake3:hash",
+ },
+ {
+ safe.RelPath("example.tar.gz"): 100,
+ safe.RelPath("example.tar.gz.sha512"): 64,
+ },
+ ),
+ ),
+ mock.patch.object(revision.attestable, "write_files_data", new=write_files_data_mock),
+ mock.patch.object(revision.attestable, "compute_file_state_rows", new=compute_file_state_rows_mock),
+ mock.patch.object(
+ revision.attestable,
+ "compute_classifications",
+ new_callable=mock.AsyncMock,
+ return_value={
+ safe.RelPath("example.tar.gz"): "source",
+ safe.RelPath("example.tar.gz.sha512"): "metadata",
+ },
+ ),
+ mock.patch.object(revision.db, "session", return_value=mock_session),
+ mock.patch.object(revision.detection, "detect_archives_requiring_quarantine", return_value=[]),
+ mock.patch.object(revision.detection, "validate_directory", return_value=[]),
+ mock.patch.object(revision.interaction, "latest_revision", new_callable=mock.AsyncMock, return_value=None),
+ mock.patch.object(revision.merge, "merge", new_callable=mock.AsyncMock),
+ mock.patch.object(revision.sql, "Revision", side_effect=_make_fake_revision),
+ mock.patch.object(revision, "SafeSession", return_value=MockSafeSession(safe_data)),
+ mock.patch.object(revision.tasks, "draft_checks", new_callable=mock.AsyncMock),
+ mock.patch.object(revision.util, "chmod_directories"),
+ mock.patch.object(revision.util, "chmod_files"),
+ mock.patch.object(revision.util, "create_hard_link_clone", new_callable=mock.AsyncMock),
+ mock.patch.object(revision.paths, "get_tmp_dir", return_value=temp_dir),
+ mock.patch.object(revision.paths, "get_archives_dir", return_value=temp_dir),
+ mock.patch.object(revision.util, "paths_to_inodes", return_value={}),
+ mock.patch.object(revision.paths, "release_directory", return_value=temp_dir / "releases" / "00001"),
+ mock.patch.object(revision.paths, "release_directory_base", return_value=temp_dir / "releases"),
+ ]
+
+ with contextlib.ExitStack() as stack:
+ for patch in patches:
+ stack.enter_context(patch)
+ await participant.create_revision_with_quarantine(
+ safe.ProjectKey("proj"),
+ safe.VersionKey("1.0"),
+ "test",
+ allowed_phases=frozenset({sql.ReleasePhase.RELEASE_CANDIDATE_DRAFT}),
+ modify=modify,
+ )
+
+ assert write_files_data_mock.await_count == 1
+ forwarded = write_files_data_mock.await_args.kwargs.get("effective_path_provenance")
+ assert forwarded == {"example.tar.gz.sha512": provenance}
+ assert compute_file_state_rows_mock.call_count == 1
+ forwarded_rows = compute_file_state_rows_mock.call_args.kwargs.get("effective_path_provenance")
+ assert forwarded_rows == {"example.tar.gz.sha512": provenance}
+
+
@pytest.mark.asyncio
async def test_v1_previous_attestable_suppresses_file_state_rows(tmp_path: pathlib.Path):
temp_dir = safe.StatePath(tmp_path)
diff --git a/tests/unit/test_release_file_state.py b/tests/unit/test_release_file_state.py
index dfcbcb1c..2b0b3954 100644
--- a/tests/unit/test_release_file_state.py
+++ b/tests/unit/test_release_file_state.py
@@ -84,6 +84,33 @@ def test_compute_file_state_rows_changed_hash():
assert rows[0].classification == "metadata"
+def test_compute_file_state_rows_changed_provenance():
+ provenance = models.ProvenanceV2(
+ generator=models.GeneratorV2.SHA512_FROM_SIGNATURE,
+ metadata={"initiated_by": "alice", "source_paths": ["app.tar.gz"]},
+ )
+ previous = models.AttestableV2(
+ paths={
+ "app.tar.gz.sha512": models.PathEntryV2(content_hash="hash1", classification="metadata"),
+ },
+ )
+ rows = attestable.compute_file_state_rows(
+ "example-0.0.1",
+ 2,
+ {"app.tar.gz.sha512": "hash1"},
+ {"app.tar.gz.sha512": "metadata"},
+ previous,
+ effective_path_provenance={"app.tar.gz.sha512": provenance},
+ )
+
+ assert len(rows) == 1
+ assert rows[0].path == "app.tar.gz.sha512"
+ assert rows[0].present is True
+ assert rows[0].content_hash == "hash1"
+ assert rows[0].classification == "metadata"
+ assert rows[0].provenance == provenance.model_dump(mode="json")
+
+
def test_compute_file_state_rows_deleted_path():
previous = models.AttestableV2(
paths={
@@ -97,6 +124,26 @@ def test_compute_file_state_rows_deleted_path():
assert rows[0].present is False
assert rows[0].content_hash is None
assert rows[0].classification is None
+ assert rows[0].provenance is None
+
+
+def test_compute_file_state_rows_deleted_path_with_previous_provenance():
+ provenance = models.ProvenanceV2(
+ generator=models.GeneratorV2.SHA512_FROM_SIGNATURE,
+ metadata={"initiated_by": "alice", "source_paths": ["old-file.tar.gz"]},
+ )
+ previous = models.AttestableV2(
+ paths={
+ "old-file.tar.gz.sha512": models.PathEntryV2(
+ content_hash="hash1", classification="metadata", provenance=provenance
+ ),
+ },
+ )
+ rows = attestable.compute_file_state_rows("example-0.0.1", 2, {}, {}, previous)
+
+ assert len(rows) == 1
+ assert rows[0].present is False
+ assert rows[0].provenance is None
def test_compute_file_state_rows_new_path():
@@ -166,6 +213,7 @@ def test_release_file_state_deleted():
assert state.present is False
assert state.content_hash is None
assert state.classification is None
+ assert state.provenance is None
def test_release_file_state_present():
@@ -184,3 +232,25 @@ def test_release_file_state_present():
assert state.present is True
assert state.content_hash == "blake3:7f83b1657ff1fc"
assert state.classification == "source"
+ assert state.provenance is None
+
+
+def test_release_file_state_present_with_provenance():
+ provenance_payload = {
+ "generator": "SHA512_from_signature",
+ "metadata": {
+ "initiated_by": "alice",
+ "source_paths": ["apache-example-0.0.1-src.tar.gz"],
+ },
+ }
+ state = sql.ReleaseFileState(
+ release_key="example-0.0.1",
+ path="apache-example-0.0.1-src.tar.gz.sha512",
+ since_revision_seq=1,
+ present=True,
+ content_hash="blake3:def",
+ classification="metadata",
+ provenance=provenance_payload,
+ )
+
+ assert state.provenance == provenance_payload
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org