You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/09/08 08:42:50 UTC
(tooling-trusted-releases) branch arm updated (f64a6374 -> 482a3c5c)
This is an automated email from the ASF dual-hosted git repository.
alitheg pushed a change to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
omit f64a6374 #1572 - add all tables with sensitive redactions
new 482a3c5c #1572 - add all tables with sensitive redactions
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f64a6374)
\
N -- N -- N refs/heads/arm (482a3c5c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
atr/admin/__init__.py | 5 ++---
tests/unit/test_admin_data_browser.py | 6 +++---
2 files changed, 5 insertions(+), 6 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-trusted-releases) 01/01: #1572 - add all tables with sensitive redactions
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alitheg pushed a commit to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 482a3c5c35ed750b1f0303a81e9d3c0fe00cd7b4
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Mon Sep 7 17:58:44 2026 +0100
#1572 - add all tables with sensitive redactions
---
atr/admin/__init__.py | 130 +++++++++++++++++++++++-----------
tests/unit/test_admin_data_browser.py | 67 ++++++++++++++++++
2 files changed, 155 insertions(+), 42 deletions(-)
diff --git a/atr/admin/__init__.py b/atr/admin/__init__.py
index 2dbb0f0b..f7a722bc 100644
--- a/atr/admin/__init__.py
+++ b/atr/admin/__init__.py
@@ -27,7 +27,7 @@ import pathlib
import statistics
import sys
import time
-from collections.abc import Callable, Sequence
+from collections.abc import Sequence
from typing import Annotated, Any, Final, Literal, NamedTuple
import aiofiles.os
@@ -83,6 +83,61 @@ ROUTES_MODULE: Final[Literal[True]] = True
_CERTIFICATE_FINDING_RANK: Final[dict[str, int]] = {"signing-keys": 1, "metadata": 2}
_MAXIMUM_PERMITTED_AGE_DAYS: Final = datetime.timedelta(days=90)
+# Every table the data browser will show. This is the whole set on purpose - the
+# _browsable_models_are_complete test fails if a new table=True model is added
+# without landing here, so the browser can't silently fall behind the schema.
+_BROWSABLE_MODELS: Final[tuple[type[sqlmodel.SQLModel], ...]] = (
+ sql.ApprovalRequest,
+ sql.Artifact,
+ sql.BallotPaper,
+ sql.Banner,
+ sql.CheckResult,
+ sql.CheckResultIgnore,
+ sql.Committee,
+ sql.Distribution,
+ sql.KeyAttestable,
+ sql.KeyLink,
+ sql.LifecycleEvent,
+ sql.Notification,
+ sql.PersonalAccessToken,
+ sql.Project,
+ sql.ProjectCycle,
+ sql.PubSubFailure,
+ sql.Quarantined,
+ sql.Release,
+ sql.ReleaseFileState,
+ sql.ReleasePolicy,
+ sql.Revision,
+ sql.RevisionCounter,
+ sql.SessionFormError,
+ sql.SignatureHint,
+ sql.SigningCertificate,
+ sql.SigningKey,
+ sql.SSHKey,
+ sql.Task,
+ sql.TextValue,
+ sql.User,
+ sql.UserSession,
+ sql.VoteCounter,
+ sql.WorkflowJti,
+ sql.WorkflowSSHKey,
+ sql.WorkflowStatus,
+)
+
+# Columns holding a secret we mustn't dump in full. The browser shows a short
+# prefix of each value and ellipsises the rest - enough to recognise the row,
+# never enough to reuse the secret. Public hashes (content, commit, dedup) are
+# left out on purpose, since they give nothing away.
+_SENSITIVE_COLUMNS: Final[dict[str, frozenset[str]]] = {
+ "PersonalAccessToken": frozenset({"token_hash"}),
+ "Quarantined": frozenset({"token"}),
+ "SessionFormError": frozenset({"sid_hash"}),
+ "UserSession": frozenset({"sid_hash"}),
+}
+
+# How much of a redacted value we reveal before the ellipsis.
+_REDACTION_VISIBLE_PREFIX: Final = 12
+
type BANNER_RESTORE = Literal["BANNER_RESTORE"]
type BANNER_SET = Literal["BANNER_SET"]
@@ -2114,56 +2169,27 @@ async def _database_consistency_tab() -> htm.Element:
async def _database_data_tab(model: str, query_args: web.PageQuery) -> htm.Element:
"""Browse all records in the database."""
- async with db.session() as data:
- # Map of model names to their classes
- # TODO: Add distribution channel, key link, and any others
- model_methods: dict[str, Callable[[], db.Query[Any]]] = {
- "CheckResult": data.check_result,
- "CheckResultIgnore": data.check_result_ignore,
- "Committee": data.committee,
- "Project": data.project,
- "PubSubFailure": data.pub_sub_failure,
- "SigningCertificate": lambda: data.signing_certificate(deleted=db.NOT_SET),
- "Release": data.release,
- "ReleasePolicy": data.release_policy,
- "Revision": data.revision,
- "SSHKey": data.ssh_key,
- "Task": data.task,
- "TextValue": data.text_value,
- }
+ models_by_name = {model_class.__name__: model_class for model_class in _BROWSABLE_MODELS}
+ if model not in models_by_name:
+ raise base.ASFQuartException(f"Model type '{model}' not found", 404)
+ model_class = models_by_name[model]
- if model not in model_methods:
- raise base.ASFQuartException(f"Model type '{model}' not found", 404)
-
- # Get all records for the selected model
- query = model_methods[model]()
- model_class = query.query.column_descriptions[0]["type"]
+ async with db.session() as data:
+ # A plain select over the class, so every table can be browsed without its own
+ # accessor. Rows aren't filtered here, so soft-deleted ones show up too, which is
+ # what an admin wants when eyeballing the raw database.
count_result = await data.execute(sqlalchemy.select(sqlalchemy.func.count()).select_from(model_class))
count = count_result.scalar_one()
- query = query.order_by(*sqlalchemy.inspect(model_class).primary_key)
+ query = db.Query(data, sqlmodel.select(model_class)).order_by(*sqlalchemy.inspect(model_class).primary_key)
records = await query.limit(query_args.limit).offset(query_args.offset).all()
- # Convert records to dictionaries for JSON serialization
- records_dict = []
- for record in records:
- if hasattr(record, "dict"):
- record_dict = record.dict()
- else:
- # Fallback for models without dict() method
- record_dict = {}
- # record_dict = {
- # "id": getattr(record, "id", None),
- # "name": getattr(record, "name", None),
- # }
- for key in record.__dict__:
- if not key.startswith("_"):
- record_dict[key] = getattr(record, key)
- records_dict.append(record_dict)
+ sensitive = _SENSITIVE_COLUMNS.get(model, frozenset())
+ records_dict = [_record_to_dict(record, sensitive) for record in records]
page = web.page_nav(query_args.offset, query_args.limit, count, len(records))
content = await template.render(
"data-browser.html",
- models=list(model_methods.keys()),
+ models=list(models_by_name.keys()),
model=model,
records=records_dict,
count=count,
@@ -2375,6 +2401,26 @@ def _keys_update_gated() -> bool:
return util.svn_publish_target() is util.SvnPublishTarget.RELEASE
+def _record_to_dict(record: Any, sensitive: frozenset[str]) -> dict[str, Any]:
+ if hasattr(record, "dict"):
+ record_dict = record.dict()
+ else:
+ # Fallback for anything without a dict() method, keeping public columns only
+ record_dict = {key: getattr(record, key) for key in record.__dict__ if not key.startswith("_")}
+ for column in sensitive:
+ if column in record_dict:
+ record_dict[column] = _redact(record_dict[column])
+ return record_dict
+
+
+def _redact(value: Any) -> Any:
+ if not isinstance(value, str):
+ return value
+ if len(value) <= _REDACTION_VISIBLE_PREFIX:
+ return "..."
+ return f"{value[:_REDACTION_VISIBLE_PREFIX]}..."
+
+
def _release_age_row(release: sql.Release, now: datetime.datetime) -> ReleaseAgeRow:
age = now - release.created
inactive_days = release.days_since_active
diff --git a/tests/unit/test_admin_data_browser.py b/tests/unit/test_admin_data_browser.py
new file mode 100644
index 00000000..26bce226
--- /dev/null
+++ b/tests/unit/test_admin_data_browser.py
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy
+import sqlmodel
+
+import atr.admin as admin
+
+
+def _table_models() -> set[type]:
+ # Every mapped table class, walked from the SQLModel base so we don't have to
+ # keep our own list in step - that's the whole point of the guard below.
+ discovered: set[type] = set()
+ pending = list(sqlmodel.SQLModel.__subclasses__())
+ while pending:
+ model_class = pending.pop()
+ pending.extend(model_class.__subclasses__())
+ if getattr(model_class, "__table__", None) is not None:
+ discovered.add(model_class)
+ return discovered
+
+
+def test_browsable_models_are_complete() -> None:
+ # A new table=True model has to be added to the data browser, or this fails
+ missing = _table_models() - set(admin._BROWSABLE_MODELS)
+ assert missing == set(), f"tables absent from the data browser: {sorted(c.__name__ for c in missing)}"
+
+
+def test_sensitive_columns_name_real_columns() -> None:
+ # A renamed or dropped secret column would silently stop being redacted, so pin
+ # each entry to a browsable model and a column that actually exists on it
+ browsable = {model_class.__name__: model_class for model_class in admin._BROWSABLE_MODELS}
+ for model_name, columns in admin._SENSITIVE_COLUMNS.items():
+ assert model_name in browsable, f"unknown model in sensitive config: {model_name}"
+ real_columns = set(sqlalchemy.inspect(browsable[model_name]).columns.keys())
+ assert columns <= real_columns, f"{model_name} has no columns {columns - real_columns}"
+
+
+def test_redact_hides_the_bulk_of_a_secret() -> None:
+ secret = "0123456789abcdef0123456789abcdef"
+ redacted = admin._redact(secret)
+ assert redacted == secret[: admin._REDACTION_VISIBLE_PREFIX] + "..."
+ assert secret not in redacted
+ assert redacted.isascii()
+
+
+def test_redact_hides_a_short_value_whole() -> None:
+ assert admin._redact("short") == "..."
+
+
+def test_redact_passes_non_strings_through() -> None:
+ assert admin._redact(None) is None
+ assert admin._redact(42) == 42
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org