You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/03/18 10:56:11 UTC
(tooling-trusted-releases) branch github_tp_validation created (now 1c4f70a1)
This is an automated email from the ASF dual-hosted git repository.
arm pushed a change to branch github_tp_validation
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
at 1c4f70a1 #676 Validate exp and nbp when loading pydantic model for Github token. Attestable class updated to store and load model instead of dict.
This branch includes the following new commits:
new 1c4f70a1 #676 Validate exp and nbp when loading pydantic model for Github token. Attestable class updated to store and load model instead of dict.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-trusted-releases) 01/01: #676 Validate exp and nbp when loading pydantic model for Github token. Attestable class updated to store and load model instead of dict.
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arm pushed a commit to branch github_tp_validation
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 1c4f70a13c8f6261dd585a93ed7de7a85d324251
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Thu Mar 12 14:32:51 2026 +0000
#676 Validate exp and nbp when loading pydantic model for Github token. Attestable class updated to store and load model instead of dict.
---
atr/api/__init__.py | 8 ++++----
atr/attestable.py | 35 +++++++++++++++++++++++++++++++++--
atr/db/interaction.py | 17 ++++++++++-------
atr/jwtoken.py | 4 ++--
atr/models/github.py | 24 ++++++++++++++++++++++--
atr/ssh.py | 9 +++++----
atr/storage/writers/ssh.py | 13 ++++++++++---
atr/tasks/checks/compare.py | 26 +-------------------------
tests/unit/test_checks_compare.py | 21 +++++++++++----------
9 files changed, 98 insertions(+), 59 deletions(-)
diff --git a/atr/api/__init__.py b/atr/api/__init__.py
index 79ff4d95..62d03f0c 100644
--- a/atr/api/__init__.py
+++ b/atr/api/__init__.py
@@ -294,8 +294,8 @@ async def distribute_ssh_register(
)
async with storage.write_as_committee_member(util.unwrap(project.committee).key, asf_uid) as wacm:
fingerprint, expires = await wacm.ssh.add_workflow_key(
- payload["actor"],
- payload["actor_id"],
+ payload.actor,
+ payload.actor_id,
release.safe_project_key,
data.ssh_key,
payload,
@@ -893,8 +893,8 @@ async def publisher_ssh_register(
)
async with storage.write_as_committee_member(util.unwrap(project.committee).key, asf_uid) as wacm:
fingerprint, expires = await wacm.ssh.add_workflow_key(
- payload["actor"],
- payload["actor_id"],
+ payload.actor,
+ payload.actor_id,
project.safe_key,
data.ssh_key,
payload,
diff --git a/atr/attestable.py b/atr/attestable.py
index fc908ad2..a58c34ae 100644
--- a/atr/attestable.py
+++ b/atr/attestable.py
@@ -29,6 +29,7 @@ import atr.classify as classify
import atr.hashes as hashes
import atr.log as log
import atr.models.attestable as models
+import atr.models.github as github
import atr.models.safe as safe
import atr.models.sql as sql
import atr.paths as paths
@@ -128,14 +129,44 @@ def github_tp_payload_path(
return paths.get_attestable_dir() / str(project_key) / str(version_key) / f"{revision_number!s}.github-tp.json"
+async def github_tp_payload_read(
+ project_key: safe.ProjectKey, version_key: safe.VersionKey, revision_number: safe.RevisionNumber
+) -> github.TrustedPublisherPayload | None:
+ payload_path = github_tp_payload_path(project_key, version_key, revision_number)
+ if not await aiofiles.os.path.isfile(payload_path):
+ return None
+ try:
+ async with aiofiles.open(payload_path, encoding="utf-8") as f:
+ data = json.loads(await f.read())
+ if not isinstance(data, dict):
+ log.warning(f"TP payload was not a JSON object in {payload_path}")
+ return None
+ # Remove exp and nbf if they're stored - as of 2026-03-18 they're validated and then removed before storage
+ # but we might have older data
+ if "exp" in data:
+ del data["exp"]
+ if "nbf" in data:
+ del data["nbf"]
+ return github.TrustedPublisherPayload.model_validate(data)
+ except (OSError, json.JSONDecodeError) as e:
+ log.warning(f"Failed to read TP payload from {payload_path}: {e}")
+ return None
+ except pydantic.ValidationError as e:
+ log.warning(f"Failed to validate TP payload from {payload_path}: {e}")
+ return None
+
+
async def github_tp_payload_write(
project_key: safe.ProjectKey,
version_key: safe.VersionKey,
revision_number: safe.RevisionNumber,
- github_payload: dict[str, Any],
+ github_payload: github.TrustedPublisherPayload,
) -> None:
payload_path = github_tp_payload_path(project_key, version_key, revision_number)
- await util.atomic_write_file(payload_path, json.dumps(github_payload, indent=2))
+ # Dump the workflow payload, excluding exp and nbf - which shouldn't have made it this far. If they do,
+ # it's safe to remove them as they've been validated by the model already, and we should never store
+ # stale dates
+ await util.atomic_write_file(payload_path, json.dumps(github_payload.model_dump(exclude={"exp", "nbf"}), indent=2))
async def load(
diff --git a/atr/db/interaction.py b/atr/db/interaction.py
index 33ef59d9..f18a1a93 100644
--- a/atr/db/interaction.py
+++ b/atr/db/interaction.py
@@ -19,7 +19,7 @@ import contextlib
import datetime
import enum
from collections.abc import AsyncGenerator, Sequence
-from typing import Any, Final
+from typing import Final
import packaging.version as version
import sqlalchemy
@@ -31,6 +31,7 @@ import atr.db as db
import atr.jwtoken as jwtoken
import atr.ldap as ldap
import atr.log as log
+import atr.models.github as github
import atr.models.results as results
import atr.models.safe as safe
import atr.models.sql as sql
@@ -455,12 +456,14 @@ async def tasks_ongoing_revision(
return task_count, latest_revision
-async def trusted_jwt(publisher: str, jwt: str, phase: TrustedProjectPhase) -> tuple[dict[str, Any], str, sql.Project]:
+async def trusted_jwt(
+ publisher: str, jwt: str, phase: TrustedProjectPhase
+) -> tuple[github.TrustedPublisherPayload, str, sql.Project]:
payload, asf_uid = await validate_trusted_jwt(publisher, jwt)
# JWT could be for an ASF user or the trusted role, but we need a user here.
if asf_uid is None:
raise InteractionError("ASF user not found")
- project = await _trusted_project(payload["repository"], payload["workflow_ref"], phase)
+ project = await _trusted_project(payload.repository, payload.workflow_ref, phase)
return payload, asf_uid, project
@@ -471,7 +474,7 @@ async def trusted_jwt_for_dist(
phase: TrustedProjectPhase,
project_key: safe.ProjectKey,
version_key: safe.VersionKey,
-) -> tuple[dict[str, Any], str, sql.Project, sql.Release]:
+) -> tuple[github.TrustedPublisherPayload, str, sql.Project, sql.Release]:
payload, asf_uid_from_jwt = await validate_trusted_jwt(publisher, jwt)
if asf_uid_from_jwt is not None:
raise InteractionError("Must use Trusted Publishing when specifying ASF UID")
@@ -553,12 +556,12 @@ async def user_projects(asf_uid: str, caller_data: db.Session | None = None) ->
return [(p.key, p.display_name) for p in projects]
-async def validate_trusted_jwt(publisher: str, jwt: str) -> tuple[dict[str, Any], str | None]:
+async def validate_trusted_jwt(publisher: str, jwt: str) -> tuple[github.TrustedPublisherPayload, str | None]:
if publisher != "github":
raise InteractionError(f"Publisher {publisher} not supported")
payload = await jwtoken.verify_github_oidc(jwt)
- if int(payload["actor_id"]) != _GITHUB_TRUSTED_ROLE_NID:
- asf_uid = await ldap.github_to_apache(payload["actor_id"])
+ if payload.actor_id != _GITHUB_TRUSTED_ROLE_NID:
+ asf_uid = await ldap.github_to_apache(payload.actor_id)
else:
asf_uid = None
return payload, asf_uid
diff --git a/atr/jwtoken.py b/atr/jwtoken.py
index 7b9700df..38e00d32 100644
--- a/atr/jwtoken.py
+++ b/atr/jwtoken.py
@@ -154,7 +154,7 @@ async def verify(token: str) -> dict[str, Any]:
return claims
-async def verify_github_oidc(token: str) -> dict[str, Any]:
+async def verify_github_oidc(token: str) -> github.TrustedPublisherPayload:
header = jwt.get_unverified_header(token)
dangerous_headers = {"jku", "x5u", "jwk"}
if dangerous_headers.intersection(header.keys()):
@@ -215,7 +215,7 @@ async def verify_github_oidc(token: str) -> dict[str, Any]:
f"GitHub OIDC payload mismatch: {key} = {payload[key]} != {value}",
errorcode=401,
)
- return github.TrustedPublisherPayload.model_validate(payload).model_dump()
+ return github.TrustedPublisherPayload.model_validate(payload)
def write_new_signing_key() -> str:
diff --git a/atr/models/github.py b/atr/models/github.py
index d20ac73f..e5fe39fc 100644
--- a/atr/models/github.py
+++ b/atr/models/github.py
@@ -17,19 +17,23 @@
from __future__ import annotations
+import time
+
+import pydantic
+
from . import schema
class TrustedPublisherPayload(schema.Subset):
actor: str
- actor_id: str
+ actor_id: int
aud: str
base_ref: str
check_run_id: str
enterprise: str
enterprise_id: str
event_name: str
- exp: int
+ exp: int | None = None
head_ref: str
iat: int
iss: str
@@ -51,3 +55,19 @@ class TrustedPublisherPayload(schema.Subset):
workflow: str
workflow_ref: str
workflow_sha: str
+
+ @pydantic.field_validator("exp")
+ @classmethod
+ def _validate_exp(cls, value: int) -> int:
+ now = int(time.time())
+ if now > value:
+ raise ValueError("Token has expired")
+ return value
+
+ @pydantic.field_validator("nbf")
+ @classmethod
+ def _validate_nbf(cls, value: int | None) -> int | None:
+ now = int(time.time())
+ if value and now < value:
+ raise ValueError("Token not yet valid")
+ return value
diff --git a/atr/ssh.py b/atr/ssh.py
index 2b78de79..533fc86f 100644
--- a/atr/ssh.py
+++ b/atr/ssh.py
@@ -26,7 +26,7 @@ import pathlib
import stat
import string
import time
-from typing import Any, Final
+from typing import Final
import aiofiles
import aiofiles.os
@@ -40,6 +40,7 @@ import atr.attestable as attestable
import atr.config as config
import atr.db as db
import atr.log as log
+import atr.models.github as github
import atr.models.safe as safe
import atr.models.sql as sql
import atr.paths as paths
@@ -82,7 +83,7 @@ class SSHServer(asyncssh.SSHServer):
# Store connection for use in begin_auth
self._conn = conn
self._github_asf_uid: str | None = None
- self._github_payload: dict[str, Any] | None = None
+ self._github_payload: github.TrustedPublisherPayload | None = None
peer_addr = conn.get_extra_info("peername")[0]
log.info(f"SSH connection received from {peer_addr}")
@@ -163,7 +164,7 @@ class SSHServer(asyncssh.SSHServer):
log.failed_authentication("public_key_expired")
return False
- self._github_payload = workflow_key.github_payload
+ self._github_payload = github.TrustedPublisherPayload.model_validate(workflow_key.github_payload)
return True
def _get_asf_uid(self, process: asyncssh.SSHServerProcess) -> str:
@@ -174,7 +175,7 @@ class SSHServer(asyncssh.SSHServer):
return self._github_asf_uid
return username
- def _get_github_payload(self, process: asyncssh.SSHServerProcess) -> dict[str, Any] | None:
+ def _get_github_payload(self, process: asyncssh.SSHServerProcess) -> github.TrustedPublisherPayload | None:
username = process.get_extra_info("username")
if username != "github":
return None
diff --git a/atr/storage/writers/ssh.py b/atr/storage/writers/ssh.py
index 5cfb5e28..fb776e4a 100644
--- a/atr/storage/writers/ssh.py
+++ b/atr/storage/writers/ssh.py
@@ -19,9 +19,9 @@
from __future__ import annotations
import time
-from typing import Any
import atr.db as db
+import atr.models.github as github
import atr.models.safe as safe
import atr.models.sql as sql
import atr.storage as storage
@@ -86,13 +86,20 @@ class CommitteeParticipant(FoundationCommitter):
self.__committee_key = committee_key
async def add_workflow_key(
- self, github_uid: str, github_nid: int, project_key: safe.ProjectKey, key: str, github_payload: dict[str, Any]
+ self,
+ github_uid: str,
+ github_nid: int,
+ project_key: safe.ProjectKey,
+ key: str,
+ github_payload: github.TrustedPublisherPayload,
) -> tuple[str, int]:
now = int(time.time())
# Twenty minutes to upload all files
ttl = 20 * 60
expires = now + ttl
fingerprint = util.key_ssh_fingerprint(key)
+ # Exclude nbf and exp as we've already validated this key - now protected by workflowkey "expires"
+ json_payload = github_payload.model_dump(exclude={"exp", "nbf"})
wsk = sql.WorkflowSSHKey(
fingerprint=fingerprint,
key=key,
@@ -100,7 +107,7 @@ class CommitteeParticipant(FoundationCommitter):
asf_uid=self.__asf_uid,
github_uid=github_uid,
github_nid=github_nid,
- github_payload=github_payload,
+ github_payload=json_payload,
expires=expires,
)
self.__data.add(wsk)
diff --git a/atr/tasks/checks/compare.py b/atr/tasks/checks/compare.py
index 73264b64..ace6f52a 100644
--- a/atr/tasks/checks/compare.py
+++ b/atr/tasks/checks/compare.py
@@ -18,7 +18,6 @@
import asyncio
import contextlib
import dataclasses
-import json
import os
import pathlib
import shutil
@@ -34,14 +33,12 @@ import dulwich.objects
import dulwich.objectspec
import dulwich.porcelain
import dulwich.refs
-import pydantic
import atr.attestable as attestable
import atr.config as config
import atr.log as log
import atr.models.github as github_models
import atr.models.results as results
-import atr.models.safe as safe
import atr.paths as paths
import atr.tasks.checks as checks
import atr.util as util
@@ -95,7 +92,7 @@ async def source_trees(args: checks.FunctionArguments) -> results.Results | None
)
return None
- payload = await _load_tp_payload(args.project_key, args.version_key, args.revision_number)
+ payload = await attestable.github_tp_payload_read(args.project_key, args.version_key, args.revision_number)
checkout_dir: str | None = None
archive_dir: str | None = None
if payload is not None:
@@ -334,27 +331,6 @@ async def _find_archive_root(archive_path: pathlib.Path, extract_dir: pathlib.Pa
return ArchiveRootResult(root=found_root, extra_entries=extra_entries)
-async def _load_tp_payload(
- project_key: safe.ProjectKey, version_key: safe.VersionKey, revision_number: safe.RevisionNumber
-) -> github_models.TrustedPublisherPayload | None:
- payload_path = attestable.github_tp_payload_path(project_key, version_key, revision_number)
- if not await aiofiles.os.path.isfile(payload_path):
- return None
- try:
- async with aiofiles.open(payload_path, encoding="utf-8") as f:
- data = json.loads(await f.read())
- if not isinstance(data, dict):
- log.warning(f"TP payload was not a JSON object in {payload_path}")
- return None
- return github_models.TrustedPublisherPayload.model_validate(data)
- except (OSError, json.JSONDecodeError) as e:
- log.warning(f"Failed to read TP payload from {payload_path}: {e}")
- return None
- except pydantic.ValidationError as e:
- log.warning(f"Failed to validate TP payload from {payload_path}: {e}")
- return None
-
-
def _payload_summary(payload: github_models.TrustedPublisherPayload | None) -> dict[str, Any]:
if payload is None:
return {"present": False}
diff --git a/tests/unit/test_checks_compare.py b/tests/unit/test_checks_compare.py
index 229f3928..5e970ceb 100644
--- a/tests/unit/test_checks_compare.py
+++ b/tests/unit/test_checks_compare.py
@@ -26,6 +26,7 @@ import dulwich.objects
import dulwich.refs
import pytest
+import atr.attestable
import atr.models.github
import atr.models.sql
import atr.tasks.checks
@@ -620,7 +621,7 @@ async def test_source_trees_creates_temp_workspace_and_cleans_up(
compare = CompareRecorder(repo_only={"extra1.txt", "extra2.txt"})
tmp_root = tmp_path / "temporary-root"
- monkeypatch.setattr(atr.tasks.checks.compare, "_load_tp_payload", PayloadLoader(payload))
+ monkeypatch.setattr(atr.attestable, "github_tp_payload_read", PayloadLoader(payload))
monkeypatch.setattr(atr.tasks.checks.compare, "_checkout_github_source", checkout)
monkeypatch.setattr(atr.tasks.checks, "resolve_archive_dir", ArchiveDirResolver(cache_dir))
monkeypatch.setattr(atr.tasks.checks.compare, "_find_archive_root", find_root)
@@ -649,7 +650,7 @@ async def test_source_trees_payload_none_skips_temp_workspace(monkeypatch: pytes
recorder = RecorderStub(True)
args = _make_args(recorder)
- monkeypatch.setattr(atr.tasks.checks.compare, "_load_tp_payload", PayloadLoader(None))
+ monkeypatch.setattr(atr.attestable, "github_tp_payload_read", PayloadLoader(None))
monkeypatch.setattr(
atr.tasks.checks.compare,
"_checkout_github_source",
@@ -676,7 +677,7 @@ async def test_source_trees_permits_pkg_info_when_pyproject_toml_exists(
compare = CompareRecorder(invalid={"PKG-INFO"})
tmp_root = tmp_path / "temporary-root"
- monkeypatch.setattr(atr.tasks.checks.compare, "_load_tp_payload", PayloadLoader(payload))
+ monkeypatch.setattr(atr.attestable, "github_tp_payload_read", PayloadLoader(payload))
monkeypatch.setattr(atr.tasks.checks.compare, "_checkout_github_source", checkout)
monkeypatch.setattr(atr.tasks.checks, "resolve_archive_dir", ArchiveDirResolver(cache_dir))
monkeypatch.setattr(atr.tasks.checks.compare, "_find_archive_root", find_root)
@@ -703,7 +704,7 @@ async def test_source_trees_records_failure_when_archive_has_invalid_files(
compare = CompareRecorder(invalid={"bad1.txt", "bad2.txt"}, repo_only={"ok.txt"})
tmp_root = tmp_path / "temporary-root"
- monkeypatch.setattr(atr.tasks.checks.compare, "_load_tp_payload", PayloadLoader(payload))
+ monkeypatch.setattr(atr.attestable, "github_tp_payload_read", PayloadLoader(payload))
monkeypatch.setattr(atr.tasks.checks.compare, "_checkout_github_source", checkout)
monkeypatch.setattr(atr.tasks.checks, "resolve_archive_dir", ArchiveDirResolver(cache_dir))
monkeypatch.setattr(atr.tasks.checks.compare, "_find_archive_root", find_root)
@@ -734,7 +735,7 @@ async def test_source_trees_records_failure_when_archive_root_not_found(
find_root = FindArchiveRootRecorder(root=None)
tmp_root = tmp_path / "temporary-root"
- monkeypatch.setattr(atr.tasks.checks.compare, "_load_tp_payload", PayloadLoader(payload))
+ monkeypatch.setattr(atr.attestable, "github_tp_payload_read", PayloadLoader(payload))
monkeypatch.setattr(atr.tasks.checks.compare, "_checkout_github_source", checkout)
monkeypatch.setattr(atr.tasks.checks, "resolve_archive_dir", ArchiveDirResolver(cache_dir))
monkeypatch.setattr(atr.tasks.checks.compare, "_find_archive_root", find_root)
@@ -756,7 +757,7 @@ async def test_source_trees_records_failure_when_cache_dir_unavailable(
args = _make_args(recorder)
payload = _make_payload()
- monkeypatch.setattr(atr.tasks.checks.compare, "_load_tp_payload", PayloadLoader(payload))
+ monkeypatch.setattr(atr.attestable, "github_tp_payload_read", PayloadLoader(payload))
monkeypatch.setattr(atr.tasks.checks, "resolve_archive_dir", ArchiveDirResolver(None))
monkeypatch.setattr(
atr.tasks.checks.compare,
@@ -786,7 +787,7 @@ async def test_source_trees_records_failure_when_extra_entries_in_archive(
find_root = FindArchiveRootRecorder(root="artifact", extra_entries=["README.txt", "extra.txt"])
tmp_root = tmp_path / "temporary-root"
- monkeypatch.setattr(atr.tasks.checks.compare, "_load_tp_payload", PayloadLoader(payload))
+ monkeypatch.setattr(atr.attestable, "github_tp_payload_read", PayloadLoader(payload))
monkeypatch.setattr(atr.tasks.checks.compare, "_checkout_github_source", checkout)
monkeypatch.setattr(atr.tasks.checks, "resolve_archive_dir", ArchiveDirResolver(cache_dir))
monkeypatch.setattr(atr.tasks.checks.compare, "_find_archive_root", find_root)
@@ -817,7 +818,7 @@ async def test_source_trees_reports_repo_only_sample_limited_to_five(
compare = CompareRecorder(repo_only=repo_only_files)
tmp_root = tmp_path / "temporary-root"
- monkeypatch.setattr(atr.tasks.checks.compare, "_load_tp_payload", PayloadLoader(payload))
+ monkeypatch.setattr(atr.attestable, "github_tp_payload_read", PayloadLoader(payload))
monkeypatch.setattr(atr.tasks.checks.compare, "_checkout_github_source", checkout)
monkeypatch.setattr(atr.tasks.checks, "resolve_archive_dir", ArchiveDirResolver(cache_dir))
monkeypatch.setattr(atr.tasks.checks.compare, "_find_archive_root", find_root)
@@ -839,7 +840,7 @@ async def test_source_trees_skips_when_not_source(monkeypatch: pytest.MonkeyPatc
args = _make_args(recorder)
monkeypatch.setattr(
- atr.tasks.checks.compare, "_load_tp_payload", RaiseAsync("_load_tp_payload should not be called")
+ atr.attestable, "github_tp_payload_read", RaiseAsync("github_tp_payload_read should not be called")
)
await atr.tasks.checks.compare.source_trees(args)
@@ -871,7 +872,7 @@ def _make_payload(
"enterprise": "",
"enterprise_id": "",
"event_name": "push",
- "exp": 1,
+ "exp": 99999999999,
"head_ref": "",
"iat": 1,
"iss": "issuer",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org