You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ar...@apache.org on 2026/02/11 16:13:32 UTC
(tooling-trusted-releases) branch check_caching created (now d697a74)
This is an automated email from the ASF dual-hosted git repository.
arm pushed a change to branch check_caching
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
at d697a74 Start to move caching out of check tasks
This branch includes the following new commits:
new d697a74 Start to move caching out of check tasks
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org
(tooling-trusted-releases) 01/01: Start to move caching out of check tasks
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arm pushed a commit to branch check_caching
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit d697a74dc99f7d563ce9abc0cfb1beb3e4fb0262
Author: Alastair McFarlane <ar...@apache.org>
AuthorDate: Wed Feb 11 16:13:22 2026 +0000
Start to move caching out of check tasks
---
atr/attestable.py | 16 +--
atr/db/__init__.py | 3 +
atr/docs/checks.md | 2 +-
atr/docs/tasks.md | 2 +-
atr/file_paths.py | 28 +++++
atr/hashing.py | 42 +++++++
atr/models/sql.py | 2 +-
atr/server.py | 42 +++----
atr/storage/readers/checks.py | 3 +
atr/tasks/__init__.py | 119 +++++++++++++-----
atr/tasks/checks/__init__.py | 159 +++++++++++++++---------
atr/tasks/checks/{hashing.py => file_hash.py} | 0
atr/tasks/checks/license.py | 11 +-
migrations/versions/0049_2026.02.11_5b874ed2.py | 37 ++++++
tests/unit/recorders.py | 2 +-
15 files changed, 334 insertions(+), 134 deletions(-)
diff --git a/atr/attestable.py b/atr/attestable.py
index d4d6d15..cac950c 100644
--- a/atr/attestable.py
+++ b/atr/attestable.py
@@ -18,13 +18,13 @@
from __future__ import annotations
import json
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any
import aiofiles
import aiofiles.os
-import blake3
import pydantic
+import atr.hashing as hashing
import atr.log as log
import atr.models.attestable as models
import atr.util as util
@@ -32,8 +32,6 @@ import atr.util as util
if TYPE_CHECKING:
import pathlib
-_HASH_CHUNK_SIZE: Final[int] = 4 * 1024 * 1024
-
def attestable_path(project_name: str, version_name: str, revision_number: str) -> pathlib.Path:
return util.get_attestable_dir() / project_name / version_name / f"{revision_number}.json"
@@ -43,14 +41,6 @@ def attestable_paths_path(project_name: str, version_name: str, revision_number:
return util.get_attestable_dir() / project_name / version_name / f"{revision_number}.paths.json"
-async def compute_file_hash(path: pathlib.Path) -> str:
- hasher = blake3.blake3()
- async with aiofiles.open(path, "rb") as f:
- while chunk := await f.read(_HASH_CHUNK_SIZE):
- hasher.update(chunk)
- return f"blake3:{hasher.hexdigest()}"
-
-
def github_tp_payload_path(project_name: str, version_name: str, revision_number: str) -> pathlib.Path:
return util.get_attestable_dir() / project_name / version_name / f"{revision_number}.github-tp.json"
@@ -140,7 +130,7 @@ async def paths_to_hashes_and_sizes(directory: pathlib.Path) -> tuple[dict[str,
if "\\" in path_key:
# TODO: We should centralise this, and forbid some other characters too
raise ValueError(f"Backslash in path is forbidden: {path_key}")
- path_to_hash[path_key] = await compute_file_hash(full_path)
+ path_to_hash[path_key] = await hashing.compute_file_hash(full_path)
path_to_size[path_key] = (await aiofiles.os.stat(full_path)).st_size
return path_to_hash, path_to_size
diff --git a/atr/db/__init__.py b/atr/db/__init__.py
index eb454e0..ed4a76a 100644
--- a/atr/db/__init__.py
+++ b/atr/db/__init__.py
@@ -164,6 +164,7 @@ class Session(sqlalchemy.ext.asyncio.AsyncSession):
status: Opt[sql.CheckResultStatus] = NOT_SET,
message: Opt[str] = NOT_SET,
data: Opt[Any] = NOT_SET,
+ inputs_hash: Opt[str] = NOT_SET,
_release: bool = False,
) -> Query[sql.CheckResult]:
query = sqlmodel.select(sql.CheckResult)
@@ -188,6 +189,8 @@ class Session(sqlalchemy.ext.asyncio.AsyncSession):
query = query.where(sql.CheckResult.message == message)
if is_defined(data):
query = query.where(sql.CheckResult.data == data)
+ if is_defined(inputs_hash):
+ query = query.where(sql.CheckResult.inputs_hash == inputs_hash)
if _release:
query = query.options(joined_load(sql.CheckResult.release))
diff --git a/atr/docs/checks.md b/atr/docs/checks.md
index 7a34817..30badd0 100644
--- a/atr/docs/checks.md
+++ b/atr/docs/checks.md
@@ -52,7 +52,7 @@ This check records separate checker keys for errors, warnings, and success. Use
For each `.sha256` or `.sha512` file, ATR computes the hash of the referenced artifact and compares it with the expected value. It supports files that contain just the hash as well as files that include a filename and hash on the same line. If the suffix does not indicate `sha256` or `sha512`, the check fails.
-The checker key is `atr.tasks.checks.hashing.check`.
+The checker key is `atr.tasks.checks.file_hash.check`.
### Signature verification
diff --git a/atr/docs/tasks.md b/atr/docs/tasks.md
index 98c409a..43c8ae2 100644
--- a/atr/docs/tasks.md
+++ b/atr/docs/tasks.md
@@ -41,7 +41,7 @@ In `atr/tasks/checks` you will find several modules that perform these check tas
In `atr/tasks/__init__.py` you will see imports for existing modules where you can add an import for new check task, for example:
```python
-import atr.tasks.checks.hashing as hashing
+import atr.tasks.checks.file_hash as file_hash
import atr.tasks.checks.license as license
```
diff --git a/atr/file_paths.py b/atr/file_paths.py
new file mode 100644
index 0000000..d29d6b9
--- /dev/null
+++ b/atr/file_paths.py
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pathlib
+
+import atr.util as util
+
+
+def base_path_for_revision(project_name: str, version_name: str, revision: str) -> pathlib.Path:
+ return pathlib.Path(util.get_unfinished_dir(), project_name, version_name, revision)
+
+
+def revision_path_for_file(project_name: str, version_name: str, revision: str, file_name: str) -> pathlib.Path:
+ return base_path_for_revision(project_name, version_name, revision) / file_name
diff --git a/atr/hashing.py b/atr/hashing.py
new file mode 100644
index 0000000..2970e08
--- /dev/null
+++ b/atr/hashing.py
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pathlib
+from typing import Any, Final
+
+import aiofiles
+import aiofiles.os
+import blake3
+
+_HASH_CHUNK_SIZE: Final[int] = 4 * 1024 * 1024
+
+
+async def compute_file_hash(path: str | pathlib.Path) -> str:
+ path = pathlib.Path(path)
+ hasher = blake3.blake3()
+ async with aiofiles.open(path, "rb") as f:
+ while chunk := await f.read(_HASH_CHUNK_SIZE):
+ hasher.update(chunk)
+ return f"blake3:{hasher.hexdigest()}"
+
+
+def compute_dict_hash(to_hash: dict[Any, Any]) -> str:
+ hasher = blake3.blake3()
+ for k in sorted(to_hash.keys()):
+ hasher.update(str(k).encode("utf-8"))
+ hasher.update(str(to_hash[k]).encode("utf-8"))
+ return f"blake3:{hasher.hexdigest()}"
diff --git a/atr/models/sql.py b/atr/models/sql.py
index a6e9aed..c03a395 100644
--- a/atr/models/sql.py
+++ b/atr/models/sql.py
@@ -946,7 +946,7 @@ class CheckResult(sqlmodel.SQLModel, table=True):
data: Any = sqlmodel.Field(
sa_column=sqlalchemy.Column(sqlalchemy.JSON), **example({"expected": "...", "found": "..."})
)
- input_hash: str | None = sqlmodel.Field(default=None, index=True, **example("blake3:7f83b1657ff1fc..."))
+ inputs_hash: str | None = sqlmodel.Field(default=None, index=True, **example("blake3:7f83b1657ff1fc..."))
cached: bool = sqlmodel.Field(default=False, **example(False))
diff --git a/atr/server.py b/atr/server.py
index c09357c..2499b7d 100644
--- a/atr/server.py
+++ b/atr/server.py
@@ -437,27 +437,27 @@ def _app_setup_request_lifecycle(app: base.QuartApp) -> None:
# Check if session has a creation timestamp in metadata
created_at_str = session.metadata.get("created_at")
- if created_at_str is None:
- # First time seeing this session, record creation time
- session.metadata["created_at"] = datetime.datetime.now(datetime.UTC).isoformat()
- pmcs = util.cookie_pmcs_or_session_pmcs(session)
- session_data = util.session_cookie_data_from_client(session, pmcs)
- util.write_quart_session_cookie(session_data)
- return
-
- # Parse the creation timestamp and check session age
- try:
- created_at = datetime.datetime.fromisoformat(created_at_str)
- except (ValueError, TypeError):
- # Invalid timestamp, treat as expired
- asfquart.session.clear()
- raise base.ASFQuartException("Session expired", errorcode=401)
-
- session_age = datetime.datetime.now(datetime.UTC) - created_at
-
- if session_age > max_lifetime:
- asfquart.session.clear()
- raise base.ASFQuartException("Session expired", errorcode=401)
+ # if created_at_str is None:
+ # # First time seeing this session, record creation time
+ # session.metadata["created_at"] = datetime.datetime.now(datetime.UTC).isoformat()
+ # pmcs = util.cookie_pmcs_or_session_pmcs(session)
+ # session_data = util.session_cookie_data_from_client(session, pmcs)
+ # util.write_quart_session_cookie(session_data)
+ # return
+
+ # # Parse the creation timestamp and check session age
+ # try:
+ # created_at = datetime.datetime.fromisoformat(created_at_str)
+ # except (ValueError, TypeError):
+ # # Invalid timestamp, treat as expired
+ # asfquart.session.clear()
+ # raise base.ASFQuartException("Session expired", errorcode=401)
+ #
+ # session_age = datetime.datetime.now(datetime.UTC) - created_at
+
+ # if session_age > max_lifetime:
+ # asfquart.session.clear()
+ # raise base.ASFQuartException("Session expired", errorcode=401)
@app.after_request
async def log_request(response: quart.Response) -> quart.Response:
diff --git a/atr/storage/readers/checks.py b/atr/storage/readers/checks.py
index b48daf3..34b181e 100644
--- a/atr/storage/readers/checks.py
+++ b/atr/storage/readers/checks.py
@@ -48,6 +48,9 @@ class GeneralPublic:
if release.latest_revision_number is None:
raise ValueError("Release has no revision - Invalid state")
+ # Consider here - we want to find by file / policy hash but we don't want to get all the task types policy keys
+ # in order to get all the possible hashes, do we?
+
query = self.__data.check_result(
release_name=release.name,
revision_number=release.latest_revision_number,
diff --git a/atr/tasks/__init__.py b/atr/tasks/__init__.py
index 8030727..34c5ce6 100644
--- a/atr/tasks/__init__.py
+++ b/atr/tasks/__init__.py
@@ -15,17 +15,21 @@
# specific language governing permissions and limitations
# under the License.
+import asyncio
import datetime
+import logging
from collections.abc import Awaitable, Callable, Coroutine
from typing import Any, Final
import sqlmodel
import atr.db as db
+import atr.hashing as hashing
import atr.models.results as results
import atr.models.sql as sql
+import atr.tasks.checks as checks
import atr.tasks.checks.compare as compare
-import atr.tasks.checks.hashing as hashing
+import atr.tasks.checks.file_hash as file_hash
import atr.tasks.checks.license as license
import atr.tasks.checks.paths as paths
import atr.tasks.checks.rat as rat
@@ -43,17 +47,20 @@ import atr.tasks.vote as vote
import atr.util as util
-async def asc_checks(asf_uid: str, release: sql.Release, revision: str, signature_path: str) -> list[sql.Task]:
+async def asc_checks(
+ asf_uid: str, release: sql.Release, revision: str, signature_path: str, data: db.Session
+) -> list[sql.Task]:
"""Create signature check task for a .asc file."""
tasks = []
if release.committee:
tasks.append(
- queued(
+ await queued(
asf_uid,
sql.TaskType.SIGNATURE_CHECK,
release,
revision,
+ data,
signature_path,
{"committee_name": release.committee.name},
)
@@ -120,9 +127,12 @@ async def draft_checks(
relative_paths = [path async for path in util.paths_recursive(revision_path)]
async with db.ensure_session(caller_data) as data:
- release = await data.release(name=sql.release_name(project_name, release_version), _committee=True).demand(
- RuntimeError("Release not found")
- )
+ release = await data.release(
+ name=sql.release_name(project_name, release_version),
+ _committee=True,
+ _release_policy=True,
+ _project_release_policy=True,
+ ).demand(RuntimeError("Release not found"))
other_releases = (
await data.release(project_name=project_name, phase=sql.ReleasePhase.RELEASE)
.order_by(sql.Release.released)
@@ -137,24 +147,26 @@ async def draft_checks(
)
for path in relative_paths:
path_str = str(path)
- task_function: Callable[[str, sql.Release, str, str], Awaitable[list[sql.Task]]] | None = None
+ task_function: Callable[[str, sql.Release, str, str, db.Session], Awaitable[list[sql.Task]]] | None = None
for suffix, func in TASK_FUNCTIONS.items():
if path.name.endswith(suffix):
task_function = func
break
if task_function:
- for task in await task_function(asf_uid, release, revision_number, path_str):
- task.revision_number = revision_number
- data.add(task)
+ for task in await task_function(asf_uid, release, revision_number, path_str, data):
+ if task:
+ task.revision_number = revision_number
+ data.add(task)
# TODO: Should we check .json files for their content?
# Ideally we would not have to do that
if path.name.endswith(".cdx.json"):
data.add(
- queued(
+ await queued(
asf_uid,
sql.TaskType.SBOM_TOOL_SCORE,
release,
revision_number,
+ caller_data,
path_str,
extra_args={
"project_name": project_name,
@@ -171,8 +183,13 @@ async def draft_checks(
if release.project.committee is not None:
if release.project.committee.is_podling:
is_podling = True
- path_check_task = queued(
- asf_uid, sql.TaskType.PATHS_CHECK, release, revision_number, extra_args={"is_podling": is_podling}
+ path_check_task = await queued(
+ asf_uid,
+ sql.TaskType.PATHS_CHECK,
+ release,
+ revision_number,
+ caller_data,
+ extra_args={"is_podling": is_podling},
)
data.add(path_check_task)
if caller_data is None:
@@ -230,14 +247,22 @@ async def metadata_update(
return task
-def queued(
+async def queued(
asf_uid: str,
task_type: sql.TaskType,
release: sql.Release,
revision_number: str,
+ data: db.Session,
primary_rel_path: str | None = None,
extra_args: dict[str, Any] | None = None,
-) -> sql.Task:
+ check_cache_key: dict[str, Any] | None = None,
+) -> sql.Task | None:
+ if check_cache_key is not None:
+ logging.info("cache key", check_cache_key)
+ hash_val = hashing.compute_dict_hash(check_cache_key)
+ existing = await data.check_result(inputs_hash=hash_val).all()
+ if existing:
+ return None
return sql.Task(
status=sql.TaskStatus.QUEUED,
task_type=task_type,
@@ -259,7 +284,7 @@ def resolve(task_type: sql.TaskType) -> Callable[..., Awaitable[results.Results
case sql.TaskType.DISTRIBUTION_WORKFLOW:
return gha.trigger_workflow
case sql.TaskType.HASHING_CHECK:
- return hashing.check
+ return file_hash.check
case sql.TaskType.KEYS_IMPORT_FILE:
return keys.import_file
case sql.TaskType.LICENSE_FILES:
@@ -304,29 +329,55 @@ def resolve(task_type: sql.TaskType) -> Callable[..., Awaitable[results.Results
# Otherwise we lose exhaustiveness checking
-async def sha_checks(asf_uid: str, release: sql.Release, revision: str, hash_file: str) -> list[sql.Task]:
+async def sha_checks(
+ asf_uid: str, release: sql.Release, revision: str, hash_file: str, data: db.Session
+) -> list[sql.Task]:
"""Create hash check task for a .sha256 or .sha512 file."""
tasks = []
- tasks.append(queued(asf_uid, sql.TaskType.HASHING_CHECK, release, revision, hash_file))
+ tasks.append(queued(asf_uid, sql.TaskType.HASHING_CHECK, release, revision, data, hash_file))
return tasks
-async def tar_gz_checks(asf_uid: str, release: sql.Release, revision: str, path: str) -> list[sql.Task]:
+async def tar_gz_checks(
+ asf_uid: str, release: sql.Release, revision: str, path: str, data: db.Session
+) -> list[sql.Task]:
"""Create check tasks for a .tar.gz or .tgz file."""
# This release has committee, as guaranteed in draft_checks
is_podling = (release.project.committee is not None) and release.project.committee.is_podling
+
tasks = [
- queued(asf_uid, sql.TaskType.COMPARE_SOURCE_TREES, release, revision, path),
- queued(asf_uid, sql.TaskType.LICENSE_FILES, release, revision, path, extra_args={"is_podling": is_podling}),
- queued(asf_uid, sql.TaskType.LICENSE_HEADERS, release, revision, path),
- queued(asf_uid, sql.TaskType.RAT_CHECK, release, revision, path),
- queued(asf_uid, sql.TaskType.TARGZ_INTEGRITY, release, revision, path),
- queued(asf_uid, sql.TaskType.TARGZ_STRUCTURE, release, revision, path),
+ queued(asf_uid, sql.TaskType.COMPARE_SOURCE_TREES, release, revision, data, path),
+ queued(
+ asf_uid,
+ sql.TaskType.LICENSE_FILES,
+ release,
+ revision,
+ data,
+ path,
+ check_cache_key=await checks.resolve_cache_key(
+ checks.license.INPUT_POLICY_KEYS, release, revision, {**{"is_podling": is_podling}}, file=path
+ ),
+ extra_args={"is_podling": is_podling},
+ ),
+ queued(
+ asf_uid,
+ sql.TaskType.LICENSE_HEADERS,
+ release,
+ revision,
+ data,
+ path,
+ check_cache_key=await checks.resolve_cache_key(
+ checks.license.INPUT_POLICY_KEYS, release, revision, file=path
+ ),
+ ),
+ queued(asf_uid, sql.TaskType.RAT_CHECK, release, revision, data, path),
+ queued(asf_uid, sql.TaskType.TARGZ_INTEGRITY, release, revision, data, path),
+ queued(asf_uid, sql.TaskType.TARGZ_STRUCTURE, release, revision, data, path),
]
- return tasks
+ return await asyncio.gather(*tasks)
async def workflow_update(
@@ -356,17 +407,19 @@ async def workflow_update(
return task
-async def zip_checks(asf_uid: str, release: sql.Release, revision: str, path: str) -> list[sql.Task]:
+async def zip_checks(asf_uid: str, release: sql.Release, revision: str, path: str, data: db.Session) -> list[sql.Task]:
"""Create check tasks for a .zip file."""
# This release has committee, as guaranteed in draft_checks
is_podling = (release.project.committee is not None) and release.project.committee.is_podling
tasks = [
- queued(asf_uid, sql.TaskType.COMPARE_SOURCE_TREES, release, revision, path),
- queued(asf_uid, sql.TaskType.LICENSE_FILES, release, revision, path, extra_args={"is_podling": is_podling}),
- queued(asf_uid, sql.TaskType.LICENSE_HEADERS, release, revision, path),
- queued(asf_uid, sql.TaskType.RAT_CHECK, release, revision, path),
- queued(asf_uid, sql.TaskType.ZIPFORMAT_INTEGRITY, release, revision, path),
- queued(asf_uid, sql.TaskType.ZIPFORMAT_STRUCTURE, release, revision, path),
+ queued(asf_uid, sql.TaskType.COMPARE_SOURCE_TREES, release, revision, data, path),
+ queued(
+ asf_uid, sql.TaskType.LICENSE_FILES, release, revision, data, path, extra_args={"is_podling": is_podling}
+ ),
+ queued(asf_uid, sql.TaskType.LICENSE_HEADERS, release, revision, data, path),
+ queued(asf_uid, sql.TaskType.RAT_CHECK, release, revision, data, path),
+ queued(asf_uid, sql.TaskType.ZIPFORMAT_INTEGRITY, release, revision, data, path),
+ queued(asf_uid, sql.TaskType.ZIPFORMAT_STRUCTURE, release, revision, data, path),
]
return tasks
diff --git a/atr/tasks/checks/__init__.py b/atr/tasks/checks/__init__.py
index 08d2c4c..9bc0fdf 100644
--- a/atr/tasks/checks/__init__.py
+++ b/atr/tasks/checks/__init__.py
@@ -20,26 +20,25 @@ from __future__ import annotations
import dataclasses
import datetime
import functools
-import pathlib
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any
import aiofiles
import aiofiles.os
-import blake3
import sqlmodel
if TYPE_CHECKING:
+ import pathlib
from collections.abc import Awaitable, Callable
import atr.models.schema as schema
import atr.config as config
import atr.db as db
+import atr.file_paths as file_paths
+import atr.hashing as hashing
import atr.models.sql as sql
import atr.util as util
-_HASH_CHUNK_SIZE: Final[int] = 4 * 1024 * 1024
-
# Pydantic does not like Callable types, so we use a dataclass instead
# It says: "you should define `Callable`, then call `FunctionArguments.model_rebuild()`"
@@ -61,7 +60,7 @@ class Recorder:
version_name: str
primary_rel_path: str | None
member_rel_path: str | None
- revision: str
+ revision_number: str
afresh: bool
__cached: bool
__input_hash: str | None
@@ -142,7 +141,7 @@ class Recorder:
message=message,
data=data,
cached=False,
- input_hash=self.__input_hash,
+ inputs_hash=self.__input_hash,
)
# It would be more efficient to keep a session open
@@ -167,7 +166,7 @@ class Recorder:
return self.abs_path_base() / rel_path_part
def abs_path_base(self) -> pathlib.Path:
- return pathlib.Path(util.get_unfinished_dir(), self.project_name, self.version_name, self.revision_number)
+ return file_paths.base_path_for_revision(self.project_name, self.version_name, self.revision_number)
async def project(self) -> sql.Project:
# TODO: Cache project
@@ -200,11 +199,8 @@ class Recorder:
abs_path = await self.abs_path()
return matches(str(abs_path))
- @property
- def cached(self) -> bool:
- return self.__cached
-
- async def check_cache(self, path: pathlib.Path) -> bool:
+ async def cache_key_set(self, policy_keys: list[str], input_args: dict[str, Any] | None = None) -> bool:
+ path = await self.abs_path()
if not await aiofiles.os.path.isfile(path):
return False
@@ -218,48 +214,74 @@ class Recorder:
if await aiofiles.os.path.exists(no_cache_file):
return False
- self.__input_hash = await _compute_file_hash(path)
-
async with db.session() as data:
- via = sql.validate_instrumented_attribute
- subquery = (
- sqlmodel.select(
- sql.CheckResult.member_rel_path,
- sqlmodel.func.max(via(sql.CheckResult.id)).label("max_id"),
- )
- .where(sql.CheckResult.checker == self.checker)
- .where(sql.CheckResult.input_hash == self.__input_hash)
- .where(sql.CheckResult.primary_rel_path == self.primary_rel_path)
- .group_by(sql.CheckResult.member_rel_path)
- .subquery()
- )
- stmt = sqlmodel.select(sql.CheckResult).join(subquery, via(sql.CheckResult.id) == subquery.c.max_id)
- results = await data.execute(stmt)
- cached_results = results.scalars().all()
-
- if not cached_results:
- return False
-
- for cached in cached_results:
- new_result = sql.CheckResult(
- release_name=self.release_name,
- revision_number=self.revision_number,
- checker=self.checker,
- primary_rel_path=self.primary_rel_path,
- member_rel_path=cached.member_rel_path,
- created=datetime.datetime.now(datetime.UTC),
- status=cached.status,
- message=cached.message,
- data=cached.data,
- cached=True,
- input_hash=self.__input_hash,
- )
- data.add(new_result)
- await data.commit()
-
- self.__cached = True
+ release = await data.release(
+ name=self.release_name, _release_policy=True, _project_release_policy=True
+ ).demand(RuntimeError(f"Release {self.release_name} not found"))
+ cache_key = await resolve_cache_key(policy_keys, release, self.revision_number, input_args, path=path)
+ self.__input_hash = hashing.compute_dict_hash(cache_key) if cache_key else None
return True
+ @property
+ def cached(self) -> bool:
+ return self.__cached
+
+ # async def check_cache(self, path: pathlib.Path) -> bool:
+ # if not await aiofiles.os.path.isfile(path):
+ # return False
+ #
+ # if config.get().DISABLE_CHECK_CACHE:
+ # return False
+ #
+ # if not await self.use_check_cache():
+ # return False
+ #
+ # no_cache_file = self.abs_path_base() / ".atr-no-cache"
+ # if await aiofiles.os.path.exists(no_cache_file):
+ # return False
+ #
+ # self.__input_hash = await hashing.compute_file_hash(path)
+ #
+ # async with db.session() as data:
+ # via = sql.validate_instrumented_attribute
+ # subquery = (
+ # sqlmodel.select(
+ # sql.CheckResult.member_rel_path,
+ # sqlmodel.func.max(via(sql.CheckResult.id)).label("max_id"),
+ # )
+ # .where(sql.CheckResult.checker == self.checker)
+ # .where(sql.CheckResult.input_hash == self.__input_hash)
+ # .where(sql.CheckResult.primary_rel_path == self.primary_rel_path)
+ # .group_by(sql.CheckResult.member_rel_path)
+ # .subquery()
+ # )
+ # stmt = sqlmodel.select(sql.CheckResult).join(subquery, via(sql.CheckResult.id) == subquery.c.max_id)
+ # results = await data.execute(stmt)
+ # cached_results = results.scalars().all()
+ #
+ # if not cached_results:
+ # return False
+ #
+ # for cached in cached_results:
+ # new_result = sql.CheckResult(
+ # release_name=self.release_name,
+ # revision_number=self.revision_number,
+ # checker=self.checker,
+ # primary_rel_path=self.primary_rel_path,
+ # member_rel_path=cached.member_rel_path,
+ # created=datetime.datetime.now(datetime.UTC),
+ # status=cached.status,
+ # message=cached.message,
+ # data=cached.data,
+ # cached=True,
+ # inputs_hash=self.__input_hash,
+ # )
+ # data.add(new_result)
+ # await data.commit()
+ #
+ # self.__cached = True
+ # return True
+
async def clear(self, primary_rel_path: str | None = None, member_rel_path: str | None = None) -> None:
async with db.session() as data:
stmt = sqlmodel.delete(sql.CheckResult).where(
@@ -348,6 +370,31 @@ def function_key(func: Callable[..., Any]) -> str:
return func.__module__ + "." + func.__name__
+async def resolve_cache_key(
+ policy_keys: list[str],
+ release: sql.Release,
+ revision: str,
+ args: dict[str, Any] | None = None,
+ file: str | None = None,
+ path: pathlib.Path | None = None,
+) -> dict[str, Any] | None:
+ if file is None and path is None:
+ raise ValueError("Must specify either file or path")
+ if not args:
+ args = {}
+ if path is None:
+ path = file_paths.revision_path_for_file(release.project_name, release.version, revision, file)
+ file_hash = await hashing.compute_file_hash(path)
+ cache_key = {"file_hash": file_hash}
+
+ policy = release.release_policy or release.project.release_policy
+ if len(policy_keys) > 0 and policy is not None:
+ policy_dict = policy.model_dump(exclude_none=True)
+ return {**cache_key, **args, **{k: policy_dict[k] for k in policy_keys if k in policy_dict}}
+ else:
+ return {**cache_key, **args}
+
+
def with_model(cls: type[schema.Strict]) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
"""Decorator to specify the parameters for a check."""
@@ -360,11 +407,3 @@ def with_model(cls: type[schema.Strict]) -> Callable[[Callable[..., Any]], Calla
return wrapper
return decorator
-
-
-async def _compute_file_hash(path: pathlib.Path) -> str:
- hasher = blake3.blake3()
- async with aiofiles.open(path, "rb") as f:
- while chunk := await f.read(_HASH_CHUNK_SIZE):
- hasher.update(chunk)
- return f"blake3:{hasher.hexdigest()}"
diff --git a/atr/tasks/checks/hashing.py b/atr/tasks/checks/file_hash.py
similarity index 100%
rename from atr/tasks/checks/hashing.py
rename to atr/tasks/checks/file_hash.py
diff --git a/atr/tasks/checks/license.py b/atr/tasks/checks/license.py
index 066fef3..3e46f11 100644
--- a/atr/tasks/checks/license.py
+++ b/atr/tasks/checks/license.py
@@ -79,6 +79,9 @@ INCLUDED_PATTERNS: Final[list[str]] = [
r"\.(pl|pm|t)$", # Perl
]
+# Release policy fields which this task relies on - used for result caching
+INPUT_POLICY_KEYS: Final[list[str]] = [""]
+
# Types
@@ -166,9 +169,11 @@ async def headers(args: checks.FunctionArguments) -> results.Results | None:
if project.policy_license_check_mode == sql.LicenseCheckMode.RAT:
return None
- if await recorder.check_cache(artifact_abs_path):
- log.info(f"Using cached license headers result for {artifact_abs_path} (rel: {args.primary_rel_path})")
- return None
+ await recorder.cache_key_set(INPUT_POLICY_KEYS)
+
+ # if await recorder.check_cache(artifact_abs_path):
+ # log.info(f"Using cached license headers result for {artifact_abs_path} (rel: {args.primary_rel_path})")
+ # return None
log.info(f"Checking license headers for {artifact_abs_path} (rel: {args.primary_rel_path})")
diff --git a/migrations/versions/0049_2026.02.11_5b874ed2.py b/migrations/versions/0049_2026.02.11_5b874ed2.py
new file mode 100644
index 0000000..e4730cd
--- /dev/null
+++ b/migrations/versions/0049_2026.02.11_5b874ed2.py
@@ -0,0 +1,37 @@
+"""Rename input_hash inputs_hash
+
+Revision ID: 0049_2026.02.11_5b874ed2
+Revises: 0048_2026.02.06_blocking_to_blocker
+Create Date: 2026-02-11 13:42:59.712570+00:00
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+# Revision identifiers, used by Alembic
+revision: str = "0049_2026.02.11_5b874ed2"
+down_revision: str | None = "0048_2026.02.06_blocking_to_blocker"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ with op.batch_alter_table("checkresult", schema=None) as batch_op:
+ batch_op.add_column(sa.Column("inputs_hash", sa.String(), nullable=True))
+ batch_op.drop_index(batch_op.f("ix_checkresult_input_hash"))
+ batch_op.create_index(batch_op.f("ix_checkresult_inputs_hash"), ["inputs_hash"], unique=False)
+ with op.batch_alter_table("checkresult", schema=None) as batch_op:
+ batch_op.execute("UPDATE checkresult SET inputs_hash = input_hash")
+ batch_op.drop_column("input_hash")
+
+
+def downgrade() -> None:
+ with op.batch_alter_table("checkresult", schema=None) as batch_op:
+ batch_op.add_column(sa.Column("input_hash", sa.VARCHAR(), nullable=True))
+ batch_op.drop_index(batch_op.f("ix_checkresult_inputs_hash"))
+ batch_op.create_index(batch_op.f("ix_checkresult_input_hash"), ["input_hash"], unique=False)
+ with op.batch_alter_table("checkresult", schema=None) as batch_op:
+ batch_op.execute("UPDATE checkresult SET input_hash = inputs_hash")
+ batch_op.drop_column("inputs_hash")
diff --git a/tests/unit/recorders.py b/tests/unit/recorders.py
index 33e5af0..47c772e 100644
--- a/tests/unit/recorders.py
+++ b/tests/unit/recorders.py
@@ -63,7 +63,7 @@ class RecorderStub(checks.Recorder):
status=status,
message=message,
data=data,
- input_hash=None,
+ inputs_hash=None,
)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org