You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2026/08/10 17:56:52 UTC
(tooling-trusted-releases) branch sbp updated: Point downloads to the release destination
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/sbp by this push:
new 75994858 Point downloads to the release destination
75994858 is described below
commit 7599485818a987e30c88e46604082bbe64a24fb4
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Mon Aug 10 18:56:03 2026 +0100
Point downloads to the release destination
---
atr/get/download.py | 76 +++++++++++++++++++++-
atr/get/file.py | 64 ++----------------
atr/shared/__init__.py | 2 +
atr/shared/published.py | 76 ++++++++++++++++++++++
atr/static/sh/download-urls.sh | 3 +-
atr/templates/download-all.html | 75 ++++++++++++---------
.../{test_get_file.py => test_shared_published.py} | 35 ++++++++--
7 files changed, 235 insertions(+), 96 deletions(-)
diff --git a/atr/get/download.py b/atr/get/download.py
index 84f55ee6..9f1e6c3d 100644
--- a/atr/get/download.py
+++ b/atr/get/download.py
@@ -34,6 +34,7 @@ import atr.mapping as mapping
import atr.models.safe as safe
import atr.models.sql as sql
import atr.paths as paths
+import atr.shared as shared
import atr.template as template
import atr.util as util
import atr.web as web
@@ -60,7 +61,13 @@ async def all_selected(
user_ssh_keys = await data.ssh_key(asf_uid=session.uid).all()
back_url = mapping.release_as_url(release)
- file_count, total_bytes, formatted_size = await util.get_release_stats(release)
+ if release.phase == sql.ReleasePhase.RELEASE:
+ published = await shared.published.release_files(release)
+ file_count = len(published)
+ total_bytes = sum((f.size or 0) for f in published)
+ formatted_size = util.format_file_size(total_bytes)
+ else:
+ file_count, total_bytes, formatted_size = await util.get_release_stats(release)
return await template.render(
"download-all.html",
@@ -147,6 +154,10 @@ async def urls_selected(
release = await data.release(project_key=str(project_key), version=str(version_key)).demand(
ValueError("Release not found")
)
+ if release.phase == sql.ReleasePhase.RELEASE:
+ published = await shared.published.release_files(release)
+ lines = [f"{f.url} {f.path}" for f in published if f.url]
+ return web.TextResponse("".join(f"{line}\n" for line in lines))
url_list_str = await _generate_file_url_list(release)
return web.TextResponse(url_list_str)
except ValueError as e:
@@ -175,6 +186,12 @@ async def zip_selected(
except Exception as e:
return web.TextResponse(_text_error(e), status=500)
+ if release.phase == sql.ReleasePhase.RELEASE:
+ return web.TextResponse(
+ "Error: Released files are served from the ASF download services, so ATR cannot provide a ZIP archive.",
+ status=404,
+ )
+
base_dir = paths.release_directory(release)
files_to_zip = []
try:
@@ -210,6 +227,8 @@ async def _download_or_list(
release = await data.release(project_key=str(project_key), version=str(version_key)).demand(
base.ASFQuartException("Release does not exist", errorcode=404)
)
+ if release.phase == sql.ReleasePhase.RELEASE:
+ return await _published_response(release, validated_path)
full_path = paths.release_directory(release) / validated_path
if await aiofiles.os.path.isdir(full_path):
@@ -297,5 +316,60 @@ async def _list(
return web.ElementResponse(response_body)
+def _published_list(
+ published: list[shared.published.PublishedFile], project_key: str, version_key: str, file_path: str
+) -> web.Response:
+ prefix = "" if (file_path == ".") else f"{file_path}/"
+ children: dict[str, str | None] = {}
+ for published_file in published:
+ if not published_file.path.startswith(prefix):
+ continue
+ remainder = published_file.path[len(prefix) :]
+ head, separator, _ = remainder.partition("/")
+ if separator:
+ children.setdefault(
+ f"{head}/",
+ util.as_url(path, project_key=project_key, version_key=version_key, file_path=f"{prefix}{head}"),
+ )
+ else:
+ children[head] = published_file.url
+ if not children:
+ return web.TextResponse("Error: File or directory not found.", status=404)
+
+ html = htm.Block(htm.html)
+ html.style["body { margin: 1rem; font: 1.25rem/1.5 serif; }"]
+ div = htm.Block()
+ if file_path != ".":
+ parent_path_str = str(pathlib.Path(file_path).parent)
+ if parent_path_str == ".":
+ parent_link_url = util.as_url(path_empty, project_key=project_key, version_key=version_key)
+ else:
+ parent_link_url = util.as_url(
+ path,
+ project_key=project_key,
+ version_key=version_key,
+ file_path=parent_path_str,
+ )
+ div.a(href=parent_link_url)["../"]
+ for name in sorted(children):
+ if (url := children[name]) is not None:
+ div.a(href=url)[name]
+ else:
+ div.span[name]
+ html.body[div.collect(separator=htm.br)]
+ return web.ElementResponse(html.collect())
+
+
+async def _published_response(release: sql.Release, validated_path: pathlib.Path) -> web.Response:
+ published = await shared.published.release_files(release)
+ path_str = str(validated_path)
+ for published_file in published:
+ if published_file.path == path_str:
+ if published_file.url is None:
+ return web.TextResponse("Error: No public download location is recorded for this file.", status=404)
+ return quart.redirect(published_file.url)
+ return _published_list(published, release.project_key, release.version, path_str)
+
+
def _text_error(error: BaseException) -> str:
return f"{errors.message(error)}\n\n{errors.traceback_text(error)}"
diff --git a/atr/get/file.py b/atr/get/file.py
index b0811886..3074ae5a 100644
--- a/atr/get/file.py
+++ b/atr/get/file.py
@@ -15,14 +15,11 @@
# specific language governing permissions and limitations
# under the License.
-import dataclasses
from typing import Final, Literal
import quart
-import atr.analysis as analysis
import atr.archives as archives
-import atr.attestable as attestable
import atr.blueprints.get as get
import atr.cycles as cycles
import atr.db as db
@@ -31,7 +28,6 @@ import atr.get.compose as compose
import atr.get.finish as finish
import atr.get.vote as vote
import atr.htm as htm
-import atr.models.attestable
import atr.models.safe as safe
import atr.models.sql as sql
import atr.paths as paths
@@ -54,13 +50,6 @@ _PHASE_BADGE_VARIANTS: Final[dict[sql.ReleasePhase, str]] = {
}
-@dataclasses.dataclass(frozen=True)
-class PublishedFile:
- path: str
- size: int | None
- url: str | None
-
-
@get.typed
async def selected(
session: web.Committer,
@@ -110,7 +99,7 @@ async def selected(
if release.phase == sql.ReleasePhase.RELEASE:
file_stats = []
- published = await _published_release_files(release, project_key, version_key)
+ published = await shared.published.release_files(release)
else:
file_stats = await _release_file_stats(release, project_key, version_key)
published = []
@@ -140,7 +129,7 @@ async def selected_path(
release = await session.release(project_key, version_key, phase=None)
if release.phase == sql.ReleasePhase.RELEASE:
- published = await _published_release_files(release, project_key, version_key)
+ published = await shared.published.release_files(release)
url = next((f.url for f in published if f.path == str(file_path)), None)
if url is not None:
return quart.redirect(url)
@@ -203,7 +192,9 @@ async def _archival_approval(release: sql.Release) -> sql.ApprovalRequest | None
).get()
-def _files_card(release: sql.Release, file_stats: list[util.FileStat], published: list[PublishedFile]) -> htm.Element:
+def _files_card(
+ release: sql.Release, file_stats: list[util.FileStat], published: list[shared.published.PublishedFile]
+) -> htm.Element:
files_card = htm.Block(htm.div, classes=".card.mb-4")
files_card.div(".card-header.d-flex.justify-content-between.align-items-center")[htm.h3(".mb-0")["Files"]]
@@ -321,51 +312,6 @@ def _phase_display_name(phase: sql.ReleasePhase) -> str:
return "release"
-def _published_files(
- artifacts: list[sql.Artifact],
- attested: atr.models.attestable.Attestable | None,
- archived: bool,
-) -> list[PublishedFile]:
- dist_dir = next((a.download_path_suffix for a in artifacts if a.download_path_suffix), None)
- sizes: dict[str, int | None] = {}
- if attested is not None:
- for path_key, content_hash in attestable.path_hashes(attested).items():
- entry = attested.hashes.get(content_hash)
- sizes[path_key] = entry.size if (entry is not None) else None
- else:
- for artifact in artifacts:
- for sibling in (
- artifact.artifact_path,
- artifact.signature_path,
- artifact.checksum_path,
- artifact.sbom_path,
- ):
- if sibling:
- sizes.setdefault(sibling, None)
- files = []
- for rel_path in sorted(sizes):
- url = None
- if dist_dir:
- is_artifact = analysis.is_artifact(rel_path)
- kind = util.DownloadFile.ARTIFACT if is_artifact else util.DownloadFile.METADATA
- url = util.download_url_for_published_path(f"{dist_dir}/{rel_path}", kind, archived=archived)
- files.append(PublishedFile(path=rel_path, size=sizes[rel_path], url=url))
- return files
-
-
-async def _published_release_files(
- release: sql.Release,
- project_key: safe.ProjectKey,
- version_key: safe.VersionKey,
-) -> list[PublishedFile]:
- async with db.session() as data:
- artifacts = list(await data.artifact(project_key=str(project_key), version=str(version_key)).all())
- attested = None
- if (revision_number := await attestable.latest_revision_number(project_key, version_key)) is not None:
- attested = await attestable.load(project_key, version_key, revision_number)
- return _published_files(artifacts, attested, release.is_archived)
-
-
async def _release_file_stats(
release: sql.Release,
project_key: safe.ProjectKey,
diff --git a/atr/shared/__init__.py b/atr/shared/__init__.py
index 3d8a269c..85adcbb1 100644
--- a/atr/shared/__init__.py
+++ b/atr/shared/__init__.py
@@ -27,6 +27,7 @@ import atr.shared.ignores as ignores
import atr.shared.keys as keys
import atr.shared.manual as manual
import atr.shared.projects as projects
+import atr.shared.published as published
import atr.shared.resolve as resolve
import atr.shared.revisions as revisions
import atr.shared.sbom as sbom
@@ -76,6 +77,7 @@ __all__ = [
"keys",
"manual",
"projects",
+ "published",
"resolve",
"revisions",
"sbom",
diff --git a/atr/shared/published.py b/atr/shared/published.py
new file mode 100644
index 00000000..b93e0a2e
--- /dev/null
+++ b/atr/shared/published.py
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import dataclasses
+
+import atr.analysis as analysis
+import atr.attestable as attestable
+import atr.db as db
+import atr.models.attestable
+import atr.models.sql as sql
+import atr.util as util
+
+
+@dataclasses.dataclass(frozen=True)
+class PublishedFile:
+ path: str
+ size: int | None
+ url: str | None
+
+
+def files(
+ artifacts: list[sql.Artifact],
+ attested: atr.models.attestable.Attestable | None,
+ archived: bool,
+) -> list[PublishedFile]:
+ entries: dict[str, tuple[int | None, str | None]] = {}
+ if attested is not None:
+ dist_dir = next((a.download_path_suffix for a in artifacts if a.download_path_suffix), None)
+ for path_key, content_hash in attestable.path_hashes(attested).items():
+ entry = attested.hashes.get(content_hash)
+ entries[path_key] = (entry.size if (entry is not None) else None, dist_dir)
+ else:
+ for artifact in artifacts:
+ for sibling in (
+ artifact.artifact_path,
+ artifact.signature_path,
+ artifact.checksum_path,
+ artifact.sbom_path,
+ ):
+ if sibling:
+ entries.setdefault(sibling, (None, artifact.download_path_suffix))
+ published = []
+ for rel_path in sorted(entries):
+ size, dist_dir = entries[rel_path]
+ url = None
+ if dist_dir:
+ is_artifact = analysis.is_artifact(rel_path)
+ kind = util.DownloadFile.ARTIFACT if is_artifact else util.DownloadFile.METADATA
+ url = util.download_url_for_published_path(f"{dist_dir}/{rel_path}", kind, archived=archived)
+ published.append(PublishedFile(path=rel_path, size=size, url=url))
+ return published
+
+
+async def release_files(release: sql.Release) -> list[PublishedFile]:
+ project_key = release.safe_project_key
+ version_key = release.safe_version_key
+ async with db.session() as data:
+ artifacts = list(await data.artifact(project_key=str(project_key), version=str(version_key)).all())
+ attested = None
+ if (revision_number := await attestable.latest_revision_number(project_key, version_key)) is not None:
+ attested = await attestable.load(project_key, version_key, revision_number)
+ return files(artifacts, attested, release.is_archived)
diff --git a/atr/static/sh/download-urls.sh b/atr/static/sh/download-urls.sh
index c7c4342e..3778d2ac 100755
--- a/atr/static/sh/download-urls.sh
+++ b/atr/static/sh/download-urls.sh
@@ -7,10 +7,11 @@ _curl_extra="[CURL_EXTRA]"
# shellcheck disable=SC2086
curl ${_curl_extra} -fsS "$_url_of_urls" | while IFS= read -r _url_and_path
do
+ [ -n "$_url_and_path" ] || continue
_url=${_url_and_path%% *}
_path=${_url_and_path#* }
printf "Downloading %s\n" "$_path" || :
# shellcheck disable=SC2086
- curl ${_curl_extra} --create-dirs -fsS "$_url" -o "$_path"
+ curl ${_curl_extra} --location --proto-redir =https --create-dirs -fsS "$_url" -o "$_path"
done
diff --git a/atr/templates/download-all.html b/atr/templates/download-all.html
index e9cbd7a4..7a395166 100644
--- a/atr/templates/download-all.html
+++ b/atr/templates/download-all.html
@@ -68,20 +68,25 @@
{% endif %}
</p>
- <h2 id="download-zip">Download ZIP archive</h2>
- <p>
- Download a single ZIP archive containing all files for this release below.
- The archive is generated on the fly, which may take a while for very large releases.
- </p>
- <p>
- <a href="{{ as_url(get.download.zip_selected, project_key=release.project.key, version_key=release.version) }}"
- class="btn btn-primary btn-lg">
- <i class="bi bi-file-earmark-zip me-2"></i>Download {{ release.key }}.zip
- </a>
- </p>
+ {% if phase != "release" %}
+ <h2 id="download-zip">Download ZIP archive</h2>
+ <p>
+ Download a single ZIP archive containing all files for this release below.
+ The archive is generated on the fly, which may take a while for very large releases.
+ </p>
+ <p>
+ <a href="{{ as_url(get.download.zip_selected, project_key=release.project.key, version_key=release.version) }}"
+ class="btn btn-primary btn-lg">
+ <i class="bi bi-file-earmark-zip me-2"></i>Download {{ release.key }}.zip
+ </a>
+ </p>
- <h2>Alternative methods</h2>
- <p>If you prefer, you can download the files using other methods.</p>
+ <h2>Alternative methods</h2>
+ <p>If you prefer, you can download the files using other methods.</p>
+ {% else %}
+ <h2>Download methods</h2>
+ <p>You can download the files using the following methods.</p>
+ {% endif %}
<p>
<a href="#download-browser"
class="btn btn-sm btn-outline-secondary me-2"
@@ -89,17 +94,23 @@
<a href="#download-curl"
class="btn btn-sm btn-outline-secondary me-2"
title="Download using curl">Curl</a>
- <a href="#download-rsync"
- class="btn btn-sm btn-outline-secondary me-2"
- title="Download using rsync">Rsync</a>
- <a href="#download-wget"
- class="btn btn-sm btn-outline-secondary me-2"
- title="Download using wget">Wget</a>
+ {% if phase != "release" %}
+ <a href="#download-rsync"
+ class="btn btn-sm btn-outline-secondary me-2"
+ title="Download using rsync">Rsync</a>
+ <a href="#download-wget"
+ class="btn btn-sm btn-outline-secondary me-2"
+ title="Download using wget">Wget</a>
+ {% endif %}
</p>
<h3 id="download-browser" class="mt-4">Using your browser</h3>
<p>
- You can download the files one by one using your browser from the <a href="{{ as_url(get.download.path_empty, project_key=release.project.key, version_key=release.version) }}">download folder</a>. Clicking a link to any file will download it, as it is served as <code>application/octet-stream</code>.
+ {% if phase != "release" %}
+ You can download the files one by one using your browser from the <a href="{{ as_url(get.download.path_empty, project_key=release.project.key, version_key=release.version) }}">download folder</a>. Clicking a link to any file will download it, as it is served as <code>application/octet-stream</code>.
+ {% else %}
+ You can download the files one by one using your browser from the <a href="{{ as_url(get.download.path_empty, project_key=release.project.key, version_key=release.version) }}">download folder</a>. Clicking a link to any file will download it from the Apache release distribution servers.
+ {% endif %}
</p>
<h3 id="download-curl" class="mt-4">Using curl</h3>
@@ -112,22 +123,24 @@ curl{% if server_domain == "localhost.apache.org" %} --insecure{% endif %} -fsS
This downloads the files into the <em>current directory</em>. Ensure that you create a new empty directory, and change to it, before running the command. The script requires curl and a POSIX compliant version of sh. It works by downloading a POSIX complaint shell script straight into your shell. You can of course <a href="{{ as_url(get.download.sh_selected, project_key=release.project.key, version_key=release.version) }}">download the script</a> and audit it before running it.
</p>
- <h3 id="download-rsync" class="mt-4">Using rsync</h3>
- <p>You can download all of the files in this release using rsync with the following command:</p>
- <!-- TODO: Add a button to copy the command to the clipboard -->
- <pre class="bg-light border rounded p-3 mb-3">
+ {% if phase != "release" %}
+ <h3 id="download-rsync" class="mt-4">Using rsync</h3>
+ <p>You can download all of the files in this release using rsync with the following command:</p>
+ <!-- TODO: Add a button to copy the command to the clipboard -->
+ <pre class="bg-light border rounded p-3 mb-3">
rsync -av -e 'ssh -p 2222' {{ asf_id }}@{{ server_domain }}:/{{ release.project.key }}/{{ release.version }}/ ${DOWNLOAD_PATH}/
</pre>
- {% include "user-ssh-keys.html" %}
+ {% include "user-ssh-keys.html" %}
- <h3 id="download-wget" class="mt-4">Using wget</h3>
- <p>You can download all of the files in this release using wget with the following command:</p>
- <pre class="bg-light border rounded p-3 mb-3">
+ <h3 id="download-wget" class="mt-4">Using wget</h3>
+ <p>You can download all of the files in this release using wget with the following command:</p>
+ <pre class="bg-light border rounded p-3 mb-3">
wget -r -np -nH --cut-dirs=4 --default-page=.index.html{% if server_domain == "localhost.apache.org" %} --no-check-certificate{% endif %} https://{{ server_host }}{{ as_url(get.download.path_empty, project_key=release.project.key, version_key=release.version) }}
</pre>
- <p>
- This downloads the files into the <em>current directory</em>. Ensure that you create a new empty directory, and change to it, before running the command.
- </p>
+ <p>
+ This downloads the files into the <em>current directory</em>. Ensure that you create a new empty directory, and change to it, before running the command.
+ </p>
+ {% endif %}
{% endblock content %}
{% block stylesheets %}
diff --git a/tests/unit/test_get_file.py b/tests/unit/test_shared_published.py
similarity index 74%
rename from tests/unit/test_get_file.py
rename to tests/unit/test_shared_published.py
index c2e79687..4d627da0 100644
--- a/tests/unit/test_get_file.py
+++ b/tests/unit/test_shared_published.py
@@ -16,9 +16,9 @@
# under the License.
import atr.constants as constants
-import atr.get.file as file_get
import atr.models.attestable as attestable_models
import atr.models.sql as sql
+import atr.shared.published as published
def test_published_files_from_artifact_rows_without_dist_dir() -> None:
@@ -32,7 +32,7 @@ def test_published_files_from_artifact_rows_without_dist_dir() -> None:
)
]
- files = file_get._published_files(artifacts, None, False)
+ files = published.files(artifacts, None, False)
assert [f.path for f in files] == [
"apache-foo-1.0.tar.gz",
@@ -54,7 +54,7 @@ def test_published_files_use_archive_host_when_archived() -> None:
)
]
- files = file_get._published_files(artifacts, None, True)
+ files = published.files(artifacts, None, True)
assert [f.url for f in files] == [
f"{constants.ARCHIVE_APACHE_URL}/tooling/foo/apache-foo-1.0.tar.gz",
@@ -86,7 +86,7 @@ def test_published_files_use_attestable_paths_and_sizes() -> None:
},
)
- files = file_get._published_files(artifacts, attested, False)
+ files = published.files(artifacts, attested, False)
assert [f.path for f in files] == ["README.md", "apache-foo-1.0.tar.gz", "apache-foo-1.0.tar.gz.asc"]
by_path = {f.path: f for f in files}
@@ -99,3 +99,30 @@ def test_published_files_use_attestable_paths_and_sizes() -> None:
== f"{constants.DOWNLOADS_APACHE_URL}/tooling/foo/apache-foo-1.0.tar.gz.asc"
)
assert by_path["README.md"].url == f"{constants.DOWNLOADS_APACHE_URL}/tooling/foo/README.md"
+
+
+def test_published_files_use_each_artifact_row_directory() -> None:
+ artifacts = [
+ sql.Artifact(
+ project_key="foo",
+ version="1.0",
+ artifact_path="apache-foo-1.0-bin.tar.gz",
+ download_path_suffix="foo/binaries",
+ ),
+ sql.Artifact(
+ project_key="foo",
+ version="1.0",
+ artifact_path="apache-foo-1.0-src.tar.gz",
+ download_path_suffix="foo/source",
+ ),
+ sql.Artifact(project_key="foo", version="1.0", artifact_path="apache-foo-1.0.pom"),
+ ]
+
+ files = published.files(artifacts, None, False)
+
+ by_path = {f.path: f for f in files}
+ binaries = f"{constants.CLOSER_LUA_URL}/foo/binaries/apache-foo-1.0-bin.tar.gz?action=download"
+ source = f"{constants.CLOSER_LUA_URL}/foo/source/apache-foo-1.0-src.tar.gz?action=download"
+ assert by_path["apache-foo-1.0-bin.tar.gz"].url == binaries
+ assert by_path["apache-foo-1.0-src.tar.gz"].url == source
+ assert by_path["apache-foo-1.0.pom"].url is None
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org