You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/08/24 10:14:40 UTC
(tooling-trusted-releases) 01/01: Don't conflict all releases on a partial import
This is an automated email from the ASF dual-hosted git repository.
alitheg pushed a commit to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 720d246b4c4f716834e318ea4637c8ce75ffc09f
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Mon Aug 24 11:14:16 2026 +0100
Don't conflict all releases on a partial import
---
atr/shared/catalogue_diff.py | 24 +++++------
atr/shared/catalogue_import.py | 28 ++++++++++---
tests/unit/test_catalogue_diff.py | 66 ++++++++++++++++++++++++++-----
tests/unit/test_catalogue_import_apply.py | 7 ++--
4 files changed, 95 insertions(+), 30 deletions(-)
diff --git a/atr/shared/catalogue_diff.py b/atr/shared/catalogue_diff.py
index 426c9606..0fb74a55 100644
--- a/atr/shared/catalogue_diff.py
+++ b/atr/shared/catalogue_diff.py
@@ -43,6 +43,9 @@ class DbSnapshot:
release_keys: frozenset[str] = dataclasses.field(default_factory=frozenset)
# The workspace committee's artifact primary keys, (project_key, version, artifact_path)
artifact_pks: frozenset[tuple[str, str, str]] = dataclasses.field(default_factory=frozenset)
+ # Release keys that hold a revision, so ATR is managing their files. Protected one by one from
+ # being repointed or catalogued over, unlike the project they sit under
+ protected_release_keys: frozenset[str] = dataclasses.field(default_factory=frozenset)
@dataclasses.dataclass(frozen=True)
@@ -198,9 +201,9 @@ def _classify_artifacts(
diff.unchanged += 1
continue
if matched is not None:
- if matched[0] in snapshot.managed_project_keys:
+ if f"{matched[0]}-{matched[1]}" in snapshot.protected_release_keys:
diff.conflicts.append(
- Conflict(table="artifacts", key=path, reason="source project has live workflow data")
+ Conflict(table="artifacts", key=path, reason="source release has live workflow data")
)
continue
if pk in claimed:
@@ -269,6 +272,10 @@ def _classify_releases(
if current_project == target_project:
diff.unchanged += 1
continue
+ if key in snapshot.protected_release_keys:
+ # A repoint rekeys the release, which would strand the files ATR holds for it
+ diff.conflicts.append(Conflict(table="releases", key=key, reason="release has live workflow data"))
+ continue
version = str(row.version)
target_release_key = f"{target_project}-{version}"
if (target_release_key in snapshot.release_keys) or (target_release_key in claimed):
@@ -308,8 +315,8 @@ def _artifact_reference_conflict(
) -> Conflict | None:
path = str(row.artifact_path)
project_key = str(row.project_key)
- if project_key in snapshot.managed_project_keys:
- return Conflict(table="artifacts", key=path, reason="target project has live workflow data")
+ if row.release_key in snapshot.protected_release_keys:
+ return Conflict(table="artifacts", key=path, reason="target release has live workflow data")
if project_key not in known_projects:
return Conflict(table="artifacts", key=path, reason=f"project '{project_key}' is not in this committee")
if row.release_key not in known_releases:
@@ -320,7 +327,8 @@ def _artifact_reference_conflict(
def _clear(
rows_by_table: dict[str, list[dict[str, str]]], snapshot: DbSnapshot, diff: CatalogueDiff, committee_key: str
) -> DbSnapshot:
- # Projects with live workflow data stay in the snapshot, and are refused as usual
+ # A project with live workflow data stays whole in the snapshot, so a replace never deletes
+ # anything under it. Its live releases are then protected one by one during classification
catalogued = {
key
for key, committee in snapshot.project_committee.items()
@@ -429,8 +437,6 @@ def _project_reference_conflict(
row: catalogue_rows.ProjectRow, snapshot: DbSnapshot, committee_key: str
) -> Conflict | None:
key = str(row.key)
- if key in snapshot.managed_project_keys:
- return Conflict(table="projects", key=key, reason="project has live workflow data")
committee = str(row.committee_key) if row.committee_key else ""
if committee != committee_key:
return Conflict(table="projects", key=key, reason=f"committee '{committee}' is not this committee")
@@ -465,14 +471,10 @@ def _release_reference_conflict(
owner = current_project if (current_project is not None) else target_project
if key != f"{owner}-{row.version}":
return Conflict(table="releases", key=key, reason=f"key does not match version '{row.version}'")
- if target_project in snapshot.managed_project_keys:
- return Conflict(table="releases", key=key, reason="target project has live workflow data")
if target_project not in known_projects:
if target_project in snapshot.project_committee:
return Conflict(table="releases", key=key, reason=f"project '{target_project}' is not in this committee")
return Conflict(table="releases", key=key, reason=f"target project '{target_project}' does not exist")
- if (current_project is not None) and (current_project in snapshot.managed_project_keys):
- return Conflict(table="releases", key=key, reason="source project has live workflow data")
return None
diff --git a/atr/shared/catalogue_import.py b/atr/shared/catalogue_import.py
index 5642340a..6ff6fda8 100644
--- a/atr/shared/catalogue_import.py
+++ b/atr/shared/catalogue_import.py
@@ -90,6 +90,7 @@ async def build_snapshot(data: db.Session, committee_key: str) -> catalogue_diff
release_project=release_project,
artifact_by_dist=artifact_by_dist,
managed_project_keys=await _managed_project_keys(data),
+ protected_release_keys=await _protected_release_keys(data),
release_keys=frozenset(release_keys),
artifact_pks=artifact_pks,
)
@@ -102,18 +103,25 @@ def discard(token: str) -> None:
async def export_table(data: db.Session, committee_key: str, table: str) -> str:
- # The import refuses to touch a project with live workflow data, so the export leaves it out.
- # Otherwise a downloaded file could not be uploaded again without editing those rows back out
- managed = await _managed_project_keys(data)
- projects = [project for project in await _committee_projects(data, committee_key) if project.key not in managed]
+ # The import refuses to touch a release with live workflow data, so the export leaves those
+ # releases out. Otherwise a downloaded file could not be uploaded again without editing them
+ # back out. The projects stay, since a replace keeps them and re-adds the rest
+ protected = await _protected_release_keys(data)
+ projects = await _committee_projects(data, committee_key)
if table == "projects":
return _to_csv(table, (_project_export(project) for project in projects))
project_keys = [project.key for project in projects]
if table == "releases":
- releases = await _committee_releases(data, project_keys)
+ releases = [
+ release for release in await _committee_releases(data, project_keys) if release.key not in protected
+ ]
return _to_csv(table, (_release_export(release) for release in releases))
if table == "artifacts":
- artifacts = await _committee_artifacts(data, project_keys)
+ artifacts = [
+ artifact
+ for artifact in await _committee_artifacts(data, project_keys)
+ if f"{artifact.project_key}-{artifact.version}" not in protected
+ ]
return _to_csv(table, (_artifact_export(artifact) for artifact in artifacts))
raise KeyError(table)
@@ -249,6 +257,14 @@ async def _managed_project_keys(data: db.Session) -> frozenset[str]:
return frozenset(row[0] for row in rows)
+async def _protected_release_keys(data: db.Session) -> frozenset[str]:
+ # A release is protected once it has a revision, which is where ATR keeps the files it holds.
+ # Foundation-wide, since a row can name a release in any committee
+ via = sql.validate_instrumented_attribute
+ rows = (await data.execute(sqlmodel.select(via(sql.Revision.release_key)).distinct())).all()
+ return frozenset(row[0] for row in rows)
+
+
def _date_str(value: datetime.datetime | None) -> str:
return value.strftime("%Y-%m-%d") if value else ""
diff --git a/tests/unit/test_catalogue_diff.py b/tests/unit/test_catalogue_diff.py
index 0590f789..a29a7ba3 100644
--- a/tests/unit/test_catalogue_diff.py
+++ b/tests/unit/test_catalogue_diff.py
@@ -50,20 +50,30 @@ def test_diff_counts_an_add() -> None:
assert d.counts["conflict"] == 0
-def test_classify_projects_adds_new_conflicts_missing_committee_and_managed() -> None:
- snap = _snapshot(managed_project_keys=frozenset({"alpha-managed"}))
+def test_classify_projects_adds_new_and_flags_a_foreign_committee() -> None:
+ snap = _snapshot()
rows = [
_project("alpha-two"), # add
_project("alpha-one"), # present -> unchanged
_project("beta-one", committee="beta"), # missing committee -> conflict
- _project("alpha-managed"), # managed -> conflict
]
d = catalogue_diff.classify_additive({"projects": rows}, snap, "alpha")
assert [str(a.key) for a in d.adds] == ["alpha-two"]
assert d.unchanged == 1
reasons = {c.key: c.reason for c in d.conflicts}
assert "beta-one" in reasons and "committee" in reasons["beta-one"].lower()
- assert "alpha-managed" in reasons
+
+
+def test_classify_projects_allows_a_project_that_owns_a_release_with_live_workflow_data() -> None:
+ # A project is kept whole rather than refused, so re-listing it is a no-op, not a conflict
+ snap = _snapshot(
+ project_committee={"alpha-live": "alpha"},
+ release_project={"alpha-live-1.0.0": "alpha-live"},
+ protected_release_keys=frozenset({"alpha-live-1.0.0"}),
+ )
+ d = catalogue_diff.classify_additive({"projects": [_project("alpha-live")]}, snap, "alpha")
+ assert not d.conflicts
+ assert d.unchanged == 1
def test_classify_releases_repoints_within_the_committee_and_refuses_to_leave_it() -> None:
@@ -155,16 +165,16 @@ def test_classify_artifacts_refuses_an_add_that_would_duplicate_an_existing_arti
assert [c.reason for c in d.conflicts] == ["artifact already exists at that project and version"]
-def test_classify_artifacts_refuses_a_repoint_out_of_a_managed_project() -> None:
+def test_classify_artifacts_refuses_a_repoint_out_of_a_release_with_live_workflow_data() -> None:
snap = _snapshot(
project_committee={"alpha-one": "alpha", "alpha-live": "alpha"},
artifact_by_dist={("alpha/1.0.0", "a.tar.gz"): ("alpha-live", "1.0.0", "a.tar.gz")},
- managed_project_keys=frozenset({"alpha-live"}),
+ protected_release_keys=frozenset({"alpha-live-1.0.0"}),
)
rows = [_artifact("alpha-one", "1.0.0", "a.tar.gz", "alpha/1.0.0")]
d = catalogue_diff.classify_additive({"artifacts": rows}, snap, "alpha")
assert not d.artifact_repoints
- assert [c.reason for c in d.conflicts] == ["source project has live workflow data"]
+ assert [c.reason for c in d.conflicts] == ["source release has live workflow data"]
def test_classify_artifacts_refuses_a_project_outside_the_committee() -> None:
@@ -441,13 +451,14 @@ def test_classify_artifacts_refuses_an_add_a_repointed_release_already_takes() -
def test_replace_refuses_a_dist_path_a_surviving_artifact_still_holds() -> None:
- # Managed projects are never cleared, so their artifacts are still catalogued afterwards. A row
- # claiming one of their dist paths would catalogue the same file in svn a second time
+ # A project with live workflow data is never cleared, so its artifacts are still catalogued
+ # afterwards. A row claiming one of their dist paths would catalogue the same file in svn twice
snap = _snapshot(
project_committee={"alpha-managed": "alpha", "alpha-two": "alpha"},
release_project={"alpha-two-1.0.0": "alpha-two"},
release_keys=frozenset({"alpha-two-1.0.0"}),
managed_project_keys=frozenset({"alpha-managed"}),
+ protected_release_keys=frozenset({"alpha-managed-1.0.0"}),
artifact_by_dist={("alpha/dist", "a.tgz"): ("alpha-managed", "1.0.0", "a.tgz")},
artifact_pks=frozenset({("alpha-managed", "1.0.0", "a.tgz")}),
)
@@ -455,4 +466,39 @@ def test_replace_refuses_a_dist_path_a_surviving_artifact_still_holds() -> None:
d = catalogue_diff.classify_replace(rows, snap, "alpha")
assert not d.adds
assert d.refused is True
- assert [c.reason for c in d.conflicts] == ["source project has live workflow data"]
+ assert [c.reason for c in d.conflicts] == ["source release has live workflow data"]
+
+
+def test_classify_releases_adds_alongside_a_release_with_live_workflow_data() -> None:
+ # The headline: a live draft protects only itself, not every release under its project
+ snap = _snapshot(
+ project_committee={"alpha-live": "alpha"},
+ release_project={"alpha-live-1.0.0": "alpha-live"},
+ release_keys=frozenset({"alpha-live-1.0.0"}),
+ protected_release_keys=frozenset({"alpha-live-1.0.0"}),
+ )
+ rows = [
+ _release("alpha-live-1.0.0", "alpha-live", "1.0.0"), # the live one -> unchanged
+ _release("alpha-live-2.0.0", "alpha-live", "2.0.0"), # new sibling -> add
+ ]
+ d = catalogue_diff.classify_additive({"releases": rows}, snap, "alpha")
+ assert [str(a.key) for a in d.adds] == ["alpha-live-2.0.0"]
+ assert d.unchanged == 1
+ assert not d.conflicts
+
+
+def test_classify_releases_refuses_repointing_a_release_with_live_workflow_data() -> None:
+ # A repoint rekeys the release, which would strand its files; a non-live sibling still moves
+ snap = _snapshot(
+ project_committee={"alpha-live": "alpha", "alpha-two": "alpha"},
+ release_project={"alpha-live-1.0.0": "alpha-live", "alpha-live-2.0.0": "alpha-live"},
+ release_keys=frozenset({"alpha-live-1.0.0", "alpha-live-2.0.0"}),
+ protected_release_keys=frozenset({"alpha-live-1.0.0"}),
+ )
+ rows = [
+ _release("alpha-live-1.0.0", "alpha-two", "1.0.0"), # repoint the live one -> refused
+ _release("alpha-live-2.0.0", "alpha-two", "2.0.0"), # repoint the sibling -> ok
+ ]
+ d = catalogue_diff.classify_additive({"releases": rows}, snap, "alpha")
+ assert [c.reason for c in d.conflicts] == ["release has live workflow data"]
+ assert [m.key for m in d.release_repoints] == ["alpha-live-2.0.0"]
diff --git a/tests/unit/test_catalogue_import_apply.py b/tests/unit/test_catalogue_import_apply.py
index 65c3a457..a6243b15 100644
--- a/tests/unit/test_catalogue_import_apply.py
+++ b/tests/unit/test_catalogue_import_apply.py
@@ -115,8 +115,9 @@ async def test_an_artifact_repoint_survives_a_repoint_of_the_release_it_came_fro
@pytest.mark.asyncio
-async def test_export_leaves_out_projects_with_live_workflow_data(sessionmaker) -> None:
- # The import refuses those rows, so an export that carried them could not be uploaded again
+async def test_export_leaves_out_releases_with_live_workflow_data(sessionmaker) -> None:
+ # The import refuses those rows, so an export that carried them could not be uploaded again.
+ # The project stays, since a replace keeps it and re-adds the rest
async with sessionmaker() as data:
await _seed_two_projects_with_release(data, {"alpha-one": "1.0.0", "alpha-two": "2.0.0"})
data.add(
@@ -135,7 +136,7 @@ async def test_export_leaves_out_projects_with_live_workflow_data(sessionmaker)
projects = [row["key"] for row in csv.DictReader(io.StringIO(projects_csv))]
releases = [row["key"] for row in csv.DictReader(io.StringIO(releases_csv))]
- assert projects == ["alpha-one"]
+ assert sorted(projects) == ["alpha-one", "alpha-two"]
assert releases == ["alpha-one-1.0.0"]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org