You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/09/07 16:59:12 UTC
(tooling-trusted-releases) branch arm updated: #1572 - add all tables with sensitive redactions
This is an automated email from the ASF dual-hosted git repository.
alitheg pushed a commit to branch arm
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/arm by this push:
new e4ac5c54 #1572 - add all tables with sensitive redactions
e4ac5c54 is described below
commit e4ac5c54b3dbd9bc1e9ec21991b20f68250931a6
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Mon Sep 7 17:58:44 2026 +0100
#1572 - add all tables with sensitive redactions
---
atr/admin/__init__.py | 131 +++++++++++++++++++++++-----------
tests/unit/test_admin_data_browser.py | 67 +++++++++++++++++
2 files changed, 156 insertions(+), 42 deletions(-)
diff --git a/atr/admin/__init__.py b/atr/admin/__init__.py
index 0f7fb09f..950de768 100644
--- a/atr/admin/__init__.py
+++ b/atr/admin/__init__.py
@@ -27,7 +27,7 @@ import pathlib
import statistics
import sys
import time
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Mapping, Sequence
from typing import Annotated, Any, Final, Literal, NamedTuple
import aiofiles.os
@@ -84,6 +84,61 @@ ROUTES_MODULE: Final[Literal[True]] = True
_CERTIFICATE_FINDING_RANK: Final[dict[str, int]] = {"signing-keys": 1, "metadata": 2}
_MAXIMUM_PERMITTED_AGE_DAYS: Final = datetime.timedelta(days=90)
+# Every table the data browser will show. This is the whole set on purpose - the
+# _browsable_models_are_complete test fails if a new table=True model is added
+# without landing here, so the browser can't silently fall behind the schema.
+_BROWSABLE_MODELS: Final[tuple[type[sqlmodel.SQLModel], ...]] = (
+ sql.ApprovalRequest,
+ sql.Artifact,
+ sql.BallotPaper,
+ sql.Banner,
+ sql.CheckResult,
+ sql.CheckResultIgnore,
+ sql.Committee,
+ sql.Distribution,
+ sql.KeyAttestable,
+ sql.KeyLink,
+ sql.LifecycleEvent,
+ sql.Notification,
+ sql.PersonalAccessToken,
+ sql.Project,
+ sql.ProjectCycle,
+ sql.PubSubFailure,
+ sql.Quarantined,
+ sql.Release,
+ sql.ReleaseFileState,
+ sql.ReleasePolicy,
+ sql.Revision,
+ sql.RevisionCounter,
+ sql.SessionFormError,
+ sql.SignatureHint,
+ sql.SigningCertificate,
+ sql.SigningKey,
+ sql.SSHKey,
+ sql.Task,
+ sql.TextValue,
+ sql.User,
+ sql.UserSession,
+ sql.VoteCounter,
+ sql.WorkflowJti,
+ sql.WorkflowSSHKey,
+ sql.WorkflowStatus,
+)
+
+# Columns holding a secret we mustn't dump in full. The browser shows a short
+# prefix of each value and ellipsises the rest - enough to recognise the row,
+# never enough to reuse the secret. Public hashes (content, commit, dedup) are
+# left out on purpose, since they give nothing away.
+_SENSITIVE_COLUMNS: Final[dict[str, frozenset[str]]] = {
+ "PersonalAccessToken": frozenset({"token_hash"}),
+ "Quarantined": frozenset({"token"}),
+ "SessionFormError": frozenset({"sid_hash"}),
+ "UserSession": frozenset({"sid_hash"}),
+}
+
+# How much of a redacted value we reveal before the ellipsis.
+_REDACTION_VISIBLE_PREFIX: Final = 12
+
type BANNER_RESTORE = Literal["BANNER_RESTORE"]
type BANNER_SET = Literal["BANNER_SET"]
@@ -2274,56 +2329,27 @@ async def _database_consistency_tab() -> htm.Element:
async def _database_data_tab(model: str, query_args: web.PageQuery) -> htm.Element:
"""Browse all records in the database."""
- async with db.session() as data:
- # Map of model names to their classes
- # TODO: Add distribution channel, key link, and any others
- model_methods: dict[str, Callable[[], db.Query[Any]]] = {
- "CheckResult": data.check_result,
- "CheckResultIgnore": data.check_result_ignore,
- "Committee": data.committee,
- "Project": data.project,
- "PubSubFailure": data.pub_sub_failure,
- "SigningCertificate": lambda: data.signing_certificate(deleted=db.NOT_SET),
- "Release": data.release,
- "ReleasePolicy": data.release_policy,
- "Revision": data.revision,
- "SSHKey": data.ssh_key,
- "Task": data.task,
- "TextValue": data.text_value,
- }
+ models_by_name = {model_class.__name__: model_class for model_class in _BROWSABLE_MODELS}
+ if model not in models_by_name:
+ raise base.ASFQuartException(f"Model type '{model}' not found", 404)
+ model_class = models_by_name[model]
- if model not in model_methods:
- raise base.ASFQuartException(f"Model type '{model}' not found", 404)
-
- # Get all records for the selected model
- query = model_methods[model]()
- model_class = query.query.column_descriptions[0]["type"]
+ async with db.session() as data:
+ # A plain select over the class, so every table can be browsed without its own
+ # accessor. Rows aren't filtered here, so soft-deleted ones show up too, which is
+ # what an admin wants when eyeballing the raw database.
count_result = await data.execute(sqlalchemy.select(sqlalchemy.func.count()).select_from(model_class))
count = count_result.scalar_one()
- query = query.order_by(*sqlalchemy.inspect(model_class).primary_key)
+ query = db.Query(data, sqlmodel.select(model_class)).order_by(*sqlalchemy.inspect(model_class).primary_key)
records = await query.limit(query_args.limit).offset(query_args.offset).all()
- # Convert records to dictionaries for JSON serialization
- records_dict = []
- for record in records:
- if hasattr(record, "dict"):
- record_dict = record.dict()
- else:
- # Fallback for models without dict() method
- record_dict = {}
- # record_dict = {
- # "id": getattr(record, "id", None),
- # "name": getattr(record, "name", None),
- # }
- for key in record.__dict__:
- if not key.startswith("_"):
- record_dict[key] = getattr(record, key)
- records_dict.append(record_dict)
+ sensitive = _SENSITIVE_COLUMNS.get(model, frozenset())
+ records_dict = [_record_to_dict(record, sensitive) for record in records]
page = web.page_nav(query_args.offset, query_args.limit, count, len(records))
content = await template.render(
"data-browser.html",
- models=list(model_methods.keys()),
+ models=list(models_by_name.keys()),
model=model,
records=records_dict,
count=count,
@@ -2477,6 +2503,27 @@ def _keys_update_gated() -> bool:
return util.svn_publish_target() is util.SvnPublishTarget.RELEASE
+def _record_to_dict(record: Any, sensitive: frozenset[str]) -> dict[str, Any]:
+ if hasattr(record, "dict"):
+ record_dict = record.dict()
+ else:
+ # Fallback for anything without a dict() method, keeping public columns only
+ record_dict = {key: getattr(record, key) for key in record.__dict__ if not key.startswith("_")}
+ for column in sensitive:
+ if column in record_dict:
+ record_dict[column] = _redact(record_dict[column])
+ return record_dict
+
+
+def _redact(value: Any) -> Any:
+ # Non-strings and short values give nothing away as a prefix, so hide them whole
+ if not isinstance(value, str):
+ return value
+ if len(value) <= _REDACTION_VISIBLE_PREFIX:
+ return "…"
+ return f"{value[:_REDACTION_VISIBLE_PREFIX]}…"
+
+
def _release_age_row(release: sql.Release, now: datetime.datetime) -> ReleaseAgeRow:
age = now - release.created
inactive_days = release.days_since_active
diff --git a/tests/unit/test_admin_data_browser.py b/tests/unit/test_admin_data_browser.py
new file mode 100644
index 00000000..014e61a8
--- /dev/null
+++ b/tests/unit/test_admin_data_browser.py
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sqlalchemy
+import sqlmodel
+
+import atr.admin as admin
+
+
+def _table_models() -> set[type]:
+ # Every mapped table class, walked from the SQLModel base so we don't have to
+ # keep our own list in step - that's the whole point of the guard below.
+ discovered: set[type] = set()
+ pending = list(sqlmodel.SQLModel.__subclasses__())
+ while pending:
+ model_class = pending.pop()
+ pending.extend(model_class.__subclasses__())
+ if getattr(model_class, "__table__", None) is not None:
+ discovered.add(model_class)
+ return discovered
+
+
+def test_browsable_models_are_complete() -> None:
+ # A new table=True model has to be added to the data browser, or this fails
+ missing = _table_models() - set(admin._BROWSABLE_MODELS)
+ assert missing == set(), f"tables absent from the data browser: {sorted(c.__name__ for c in missing)}"
+
+
+def test_sensitive_columns_name_real_columns() -> None:
+ # A renamed or dropped secret column would silently stop being redacted, so pin
+ # each entry to a browsable model and a column that actually exists on it
+ browsable = {model_class.__name__: model_class for model_class in admin._BROWSABLE_MODELS}
+ for model_name, columns in admin._SENSITIVE_COLUMNS.items():
+ assert model_name in browsable, f"unknown model in sensitive config: {model_name}"
+ real_columns = set(sqlalchemy.inspect(browsable[model_name]).columns.keys())
+ assert columns <= real_columns, f"{model_name} has no columns {columns - real_columns}"
+
+
+def test_redact_hides_the_bulk_of_a_secret() -> None:
+ secret = "0123456789abcdef0123456789abcdef"
+ redacted = admin._redact(secret)
+ assert redacted.endswith("…")
+ assert redacted[:-1] == secret[: admin._REDACTION_VISIBLE_PREFIX]
+ assert secret not in redacted
+
+
+def test_redact_hides_a_short_value_whole() -> None:
+ assert admin._redact("short") == "…"
+
+
+def test_redact_passes_non_strings_through() -> None:
+ assert admin._redact(None) is None
+ assert admin._redact(42) == 42
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org