You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2025/12/18 21:17:15 UTC
(tooling-trusted-releases) branch main updated: Add progress bars to indicate the status of uploads
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/main by this push:
new 9368da3 Add progress bars to indicate the status of uploads
9368da3 is described below
commit 9368da30bc548ef60a51ff5c4324964b2ff7d1ca
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Thu Dec 18 21:16:53 2025 +0000
Add progress bars to indicate the status of uploads
---
atr/get/upload.py | 22 ++-
atr/post/upload.py | 106 +++++++++++++-
atr/static/js/src/upload-progress-ui.js | 147 +++++++++++++++++++
atr/static/js/src/upload-progress.js | 243 ++++++++++++++++++++++++++++++++
atr/util.py | 6 +
5 files changed, 521 insertions(+), 3 deletions(-)
diff --git a/atr/get/upload.py b/atr/get/upload.py
index b60c6ef..4a86c1b 100644
--- a/atr/get/upload.py
+++ b/atr/get/upload.py
@@ -15,8 +15,11 @@
# specific language governing permissions and limitations
# under the License.
+import secrets
from collections.abc import Sequence
+import htpy
+
import atr.blueprints.get as get
import atr.db as db
import atr.form as form
@@ -63,6 +66,18 @@ async def selected(session: web.Committer, project_name: str, version_name: str)
block.h2(id="file-upload")["File upload"]
block.p["Use this form to add files to this candidate draft."]
+ upload_session_token = secrets.token_hex(16)
+ stage_url = f"/upload/stage/{project_name}/{version_name}/{upload_session_token}"
+ finalise_url = f"/upload/finalise/{project_name}/{version_name}/{upload_session_token}"
+
+ block.append(
+ htpy.div(
+ "#upload-config.atr-hide",
+ data_stage_url=stage_url,
+ data_finalise_url=finalise_url,
+ )
+ )
+
form.render_block(
block,
model_cls=shared.upload.AddFilesForm,
@@ -70,6 +85,8 @@ async def selected(session: web.Committer, project_name: str, version_name: str)
form_classes=".atr-canary.py-4.px-5",
)
+ block.append(htpy.div("#upload-progress-container.d-none"))
+
block.h2(id="svn-upload")["SVN upload"]
block.p["Import files from this project's ASF Subversion repository into this draft."]
block.p[
@@ -114,6 +131,7 @@ async def selected(session: web.Committer, project_name: str, version_name: str)
return await template.blank(
f"Upload files to {release.short_display_name}",
content=block.collect(),
+ javascripts=["upload-progress-ui", "upload-progress"],
)
@@ -136,7 +154,7 @@ def _render_ssh_keys_info(block: htm.Block, user_ssh_keys: Sequence[sql.SSHKey])
if key_count == 1:
key = user_ssh_keys[0]
key_parts = key.key.split(" ", 2)
- key_comment = key_parts[2].strip() if len(key_parts) > 2 else "key"
+ key_comment = key_parts[2].strip() if (len(key_parts) > 2) else "key"
block.p[
"We have the SSH key ",
htm.a(
@@ -152,7 +170,7 @@ def _render_ssh_keys_info(block: htm.Block, user_ssh_keys: Sequence[sql.SSHKey])
key_items = []
for key in user_ssh_keys:
key_parts = key.key.split(" ", 2)
- key_comment = key_parts[2].strip() if len(key_parts) > 2 else "key"
+ key_comment = key_parts[2].strip() if (len(key_parts) > 2) else "key"
key_items.append(
htm.li[
htm.a(
diff --git a/atr/post/upload.py b/atr/post/upload.py
index f0dc590..26cfb88 100644
--- a/atr/post/upload.py
+++ b/atr/post/upload.py
@@ -15,10 +15,16 @@
# specific language governing permissions and limitations
# under the License.
-
+import asyncio
+import json
+import pathlib
from typing import Final
+import aiofiles
+import aiofiles.os
+import aioshutil
import quart
+import werkzeug.wrappers.response as response
import atr.blueprints.post as post
import atr.db as db
@@ -26,11 +32,61 @@ import atr.get as get
import atr.log as log
import atr.shared as shared
import atr.storage as storage
+import atr.util as util
import atr.web as web
_SVN_BASE_URL: Final[str] = "https://dist.apache.org/repos/dist"
+@post.committer("/upload/finalise/<project_name>/<version_name>/<upload_session>")
+async def finalise(
+ session: web.Committer, project_name: str, version_name: str, upload_session: str
+) -> web.WerkzeugResponse:
+ await session.check_access(project_name)
+
+ try:
+ staging_dir = util.get_upload_staging_dir(upload_session)
+ except ValueError:
+ return _json_error("Invalid session token", 400)
+
+ if not await aiofiles.os.path.isdir(staging_dir):
+ return _json_error("No staged files found", 400)
+
+ try:
+ staged_files = await aiofiles.os.listdir(staging_dir)
+ except OSError:
+ return _json_error("Error reading staging directory", 500)
+
+ staged_files = [f for f in staged_files if f not in (".", "..")]
+ if not staged_files:
+ return _json_error("No staged files found", 400)
+
+ try:
+ async with storage.write(session) as write:
+ wacp = await write.as_project_committee_participant(project_name)
+ number_of_files = len(staged_files)
+ plural = "s" if number_of_files != 1 else ""
+ description = f"Upload of {number_of_files} file{plural} through web interface"
+
+ async with wacp.release.create_and_manage_revision(project_name, version_name, description) as creating:
+ for filename in staged_files:
+ src = staging_dir / filename
+ dst = creating.interim_path / filename
+ await aioshutil.move(str(src), str(dst))
+
+ await aioshutil.rmtree(staging_dir)
+
+ return await session.redirect(
+ get.compose.selected,
+ success=f"{number_of_files} file{plural} added successfully",
+ project_name=project_name,
+ version_name=version_name,
+ )
+ except Exception as e:
+ log.exception("Error finalising upload:")
+ return _json_error(f"Error finalising upload: {e!s}", 500)
+
+
@post.committer("/upload/<project_name>/<version_name>")
@post.form(shared.upload.UploadForm)
async def selected(
@@ -46,6 +102,46 @@ async def selected(
return await _svn_import(session, svn_form, project_name, version_name)
+@post.committer("/upload/stage/<project_name>/<version_name>/<upload_session>")
+async def stage(
+ session: web.Committer, project_name: str, version_name: str, upload_session: str
+) -> web.WerkzeugResponse:
+ await session.check_access(project_name)
+
+ try:
+ staging_dir = util.get_upload_staging_dir(upload_session)
+ except ValueError:
+ return _json_error("Invalid session token", 400)
+
+ files = await quart.request.files
+ file = files.get("file")
+ if (not file) or (not file.filename):
+ return _json_error("No file provided", 400)
+
+ filename = pathlib.Path(file.filename).name
+ if (not filename) or (filename in (".", "..")):
+ return _json_error("Invalid filename", 400)
+
+ await aiofiles.os.makedirs(staging_dir, exist_ok=True)
+
+ target_path = staging_dir / filename
+ if await aiofiles.os.path.exists(target_path):
+ return _json_error("File already exists in staging", 409)
+
+ try:
+ async with aiofiles.open(target_path, "wb") as f:
+ # 1 MiB chunks
+ while chunk := await asyncio.to_thread(file.stream.read, 1024 * 1024):
+ await f.write(chunk)
+ except Exception as e:
+ log.exception("Error staging file:")
+ if await aiofiles.os.path.exists(target_path):
+ await aiofiles.os.remove(target_path)
+ return _json_error(f"Error staging file: {e!s}", 500)
+
+ return _json_success({"status": "staged", "filename": filename})
+
+
async def _add_files(
session: web.Committer, add_form: shared.upload.AddFilesForm, project_name: str, version_name: str
) -> web.WerkzeugResponse:
@@ -80,6 +176,14 @@ def _construct_svn_url(project_name: str, area: shared.upload.SvnArea, path: str
return f"{_SVN_BASE_URL}/{area.value}/{project_name}/{path}"
+def _json_error(message: str, status: int) -> web.WerkzeugResponse:
+ return response.Response(json.dumps({"error": message}), status=status, mimetype="application/json")
+
+
+def _json_success(data: dict[str, str], status: int = 200) -> web.WerkzeugResponse:
+ return response.Response(json.dumps(data), status=status, mimetype="application/json")
+
+
async def _svn_import(
session: web.Committer, svn_form: shared.upload.SvnImportForm, project_name: str, version_name: str
) -> web.WerkzeugResponse:
diff --git a/atr/static/js/src/upload-progress-ui.js b/atr/static/js/src/upload-progress-ui.js
new file mode 100644
index 0000000..7648e80
--- /dev/null
+++ b/atr/static/js/src/upload-progress-ui.js
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+window.UploadUI = {
+ formatBytes(b) {
+ if (b < 1024) return `${b} B`;
+ if (b < 1048576) return `${(b / 1024).toFixed(1)} KB`;
+ if (b < 1073741824) return `${(b / 1048576).toFixed(1)} MB`;
+ return `${(b / 1073741824).toFixed(2)} GB`;
+ },
+
+ createFileProgressUI(file, index, activeUploads) {
+ const div = document.createElement("div");
+ div.className = "card mb-3";
+ div.id = `upload-file-${index}`;
+ div.innerHTML = `<div class="card-body">
+ <div class="d-flex justify-content-between align-items-start mb-2">
+ <div><strong class="file-name"></strong>
+ <small class="text-muted ms-2">(${this.formatBytes(file.size)})</small></div>
+ <button type="button" class="btn btn-sm btn-outline-secondary cancel-btn">Cancel</button>
+ </div>
+ <progress class="w-100 mb-1" value="0" max="100"></progress>
+ <div class="d-flex justify-content-between">
+ <small class="upload-status text-muted">Preparing...</small>
+ <small class="upload-percent">0%</small>
+ </div></div>`;
+ div.querySelector(".file-name").textContent = file.name;
+ div.querySelector(".cancel-btn").addEventListener("click", () => {
+ const xhr = activeUploads.get(index);
+ if (xhr) xhr.abort();
+ });
+ return div;
+ },
+
+ markUploadDone(ui, success, msg) {
+ const statusEl = ui.querySelector(".upload-status");
+ statusEl.textContent = msg;
+ statusEl.className = `upload-status text-${success ? "success" : "danger"}`;
+ const border = success ? "border-success" : "border-danger";
+ ui.querySelector(".card-body").classList.add(
+ "border-start",
+ border,
+ "border-3",
+ );
+ if (success) {
+ ui.querySelector("progress").value = 100;
+ ui.querySelector(".upload-percent").textContent = "100%";
+ }
+ ui.querySelector(".cancel-btn").classList.add("d-none");
+ },
+
+ resetUploadUI(ui) {
+ ui.querySelector(".card-body").classList.remove(
+ "border-start",
+ "border-danger",
+ "border-3",
+ );
+ const statusEl = ui.querySelector(".upload-status");
+ statusEl.textContent = "Preparing...";
+ statusEl.className = "upload-status text-muted";
+ ui.querySelector("progress").value = 0;
+ ui.querySelector(".upload-percent").textContent = "0%";
+ ui.querySelector(".cancel-btn").classList.remove("d-none");
+ },
+
+ showAllFailedSummary(container) {
+ const summary = document.createElement("div");
+ summary.className = "alert alert-danger mt-3";
+ summary.innerHTML = `<strong>All uploads failed.</strong>
+ <button type="button" class="btn btn-outline-primary ms-3" id="retry-all-btn">Try again</button>`;
+ container.append(summary);
+ document
+ .getElementById("retry-all-btn")
+ .addEventListener("click", () => location.reload());
+ },
+
+ showPartialSuccessSummary(container, completedCount, totalFiles, onFinalise) {
+ const summary = document.createElement("div");
+ summary.className = "alert alert-warning mt-3";
+ summary.id = "upload-summary";
+ const totalWord = totalFiles === 1 ? "file" : "files";
+ const stagedWord = completedCount === 1 ? "file" : "files";
+ summary.innerHTML = `<strong>${completedCount} of ${totalFiles} ${totalWord} staged.</strong>
+ <span class="ms-2">You can finalise with the staged files or retry failed uploads.</span>
+ <button type="button" class="btn btn-primary ms-3" id="finalise-btn">Finalise ${completedCount} ${stagedWord}</button>`;
+ container.append(summary);
+ document
+ .getElementById("finalise-btn")
+ .addEventListener("click", onFinalise);
+ },
+
+ doFinalise(container, finaliseUrl, csrfToken) {
+ const existingSummary = document.getElementById("upload-summary");
+ if (existingSummary) existingSummary.remove();
+ const summary = document.createElement("div");
+ summary.className = "alert alert-info mt-3";
+ summary.innerHTML = `<strong>Finalising upload...</strong>`;
+ container.append(summary);
+ const form = document.createElement("form");
+ form.method = "POST";
+ form.action = finaliseUrl;
+ form.style.display = "none";
+ const csrfInput = document.createElement("input");
+ csrfInput.type = "hidden";
+ csrfInput.name = "csrf_token";
+ csrfInput.value = csrfToken;
+ form.append(csrfInput);
+ document.body.append(form);
+ form.submit();
+ },
+
+ handleUploadProgress(e, progressBar, statusEl, percentEl, cancelBtn) {
+ if (e.lengthComputable) {
+ const pct = Math.round((e.loaded / e.total) * 100);
+ progressBar.value = pct;
+ delete progressBar.dataset.indeterminate;
+ percentEl.textContent = `${pct}%`;
+ if (pct >= 100) {
+ statusEl.textContent = "Processing...";
+ cancelBtn.classList.add("d-none");
+ } else {
+ statusEl.textContent = `${this.formatBytes(e.loaded)} / ${this.formatBytes(e.total)}`;
+ }
+ } else {
+ progressBar.removeAttribute("value");
+ progressBar.dataset.indeterminate = "true";
+ percentEl.textContent = "";
+ statusEl.textContent = `${this.formatBytes(e.loaded)} uploaded`;
+ }
+ },
+};
diff --git a/atr/static/js/src/upload-progress.js b/atr/static/js/src/upload-progress.js
new file mode 100644
index 0000000..a9ef76a
--- /dev/null
+++ b/atr/static/js/src/upload-progress.js
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+function handleXhrLoad(xhr, ctx) {
+ ctx.activeUploads.delete(ctx.index);
+ if (xhr.status >= 200 && xhr.status < 300) {
+ UploadUI.markUploadDone(ctx.ui, true, "Staged");
+ ctx.state.completedCount++;
+ } else {
+ let msg = "Upload failed";
+ try {
+ msg = JSON.parse(xhr.responseText).error || msg;
+ } catch {
+ /* ignore */
+ }
+ UploadUI.markUploadDone(ctx.ui, false, `Failed: ${msg}`);
+ ctx.state.failedCount++;
+ ctx.addRetryButton(ctx.ui, ctx.index);
+ }
+ ctx.checkAllComplete();
+}
+
+function handleXhrFailure(ctx, reason, isCancelled) {
+ ctx.activeUploads.delete(ctx.index);
+ if (isCancelled) {
+ ctx.statusEl.textContent = "Cancelled";
+ ctx.statusEl.className = "upload-status text-secondary";
+ ctx.cancelBtn.classList.add("d-none");
+ } else {
+ UploadUI.markUploadDone(ctx.ui, false, `Failed: ${reason}`);
+ ctx.addRetryButton(ctx.ui, ctx.index);
+ }
+ ctx.state.failedCount++;
+ ctx.checkAllComplete();
+}
+
+function setupXhrHandlers(
+ xhr,
+ ui,
+ index,
+ state,
+ addRetryButton,
+ checkAllComplete,
+) {
+ const ctx = {
+ ui,
+ index,
+ state,
+ addRetryButton,
+ checkAllComplete,
+ activeUploads: state.activeUploads,
+ statusEl: ui.querySelector(".upload-status"),
+ cancelBtn: ui.querySelector(".cancel-btn"),
+ };
+ const progressBar = ui.querySelector("progress");
+ const percentEl = ui.querySelector(".upload-percent");
+
+ xhr.upload.addEventListener("progress", (e) =>
+ UploadUI.handleUploadProgress(
+ e,
+ progressBar,
+ ctx.statusEl,
+ percentEl,
+ ctx.cancelBtn,
+ ),
+ );
+ xhr.addEventListener("load", () => handleXhrLoad(xhr, ctx));
+ xhr.addEventListener("error", () =>
+ handleXhrFailure(ctx, "Network error", false),
+ );
+ xhr.addEventListener("abort", () => handleXhrFailure(ctx, null, true));
+}
+
+function checkAllComplete(state, container, finaliseUrl, csrfToken) {
+ if (state.activeUploads.size > 0) return;
+ if (state.completedCount === 0) {
+ UploadUI.showAllFailedSummary(container);
+ } else if (state.failedCount > 0) {
+ UploadUI.showPartialSuccessSummary(
+ container,
+ state.completedCount,
+ state.totalFiles,
+ () => UploadUI.doFinalise(container, finaliseUrl, csrfToken),
+ );
+ } else {
+ UploadUI.doFinalise(container, finaliseUrl, csrfToken);
+ }
+}
+
+function startUpload(
+ file,
+ index,
+ state,
+ stageUrl,
+ csrfToken,
+ addRetryButton,
+ checkComplete,
+) {
+ const ui = document.getElementById(`upload-file-${index}`);
+ const xhr = new XMLHttpRequest();
+ state.activeUploads.set(index, xhr);
+ setupXhrHandlers(xhr, ui, index, state, addRetryButton, checkComplete);
+ const formData = new FormData();
+ formData.append("csrf_token", csrfToken);
+ formData.append("file", file, file.name);
+ xhr.open("POST", stageUrl, true);
+ xhr.send(formData);
+ ui.querySelector(".upload-status").textContent = "Uploading...";
+}
+
+function addRetryButton(
+ ui,
+ index,
+ state,
+ fileInput,
+ stageUrl,
+ csrfToken,
+ checkComplete,
+) {
+ const retryBtn = document.createElement("button");
+ retryBtn.type = "button";
+ retryBtn.className = "btn btn-sm btn-outline-primary retry-btn ms-2";
+ retryBtn.textContent = "Retry";
+ retryBtn.addEventListener("click", () => {
+ UploadUI.resetUploadUI(ui);
+ retryBtn.remove();
+ state.failedCount--;
+ const file = Array.from(fileInput.files || [])[index];
+ startUpload(
+ file,
+ index,
+ state,
+ stageUrl,
+ csrfToken,
+ addRetryButton,
+ checkComplete,
+ );
+ });
+ ui.querySelector(".upload-status").parentNode.append(retryBtn);
+}
+
+function handleFormSubmit(
+ e,
+ form,
+ fileInput,
+ container,
+ state,
+ stageUrl,
+ csrfToken,
+ checkComplete,
+) {
+ e.preventDefault();
+ const files = Array.from(fileInput.files || []);
+ if (files.length === 0) {
+ alert("Please select files to upload.");
+ return;
+ }
+ state.totalFiles = files.length;
+ state.completedCount = 0;
+ state.failedCount = 0;
+ form.classList.add("d-none");
+ container.classList.remove("d-none");
+ container.innerHTML = "";
+ files.forEach((file, index) => {
+ container.append(
+ UploadUI.createFileProgressUI(file, index, state.activeUploads),
+ );
+ startUpload(
+ file,
+ index,
+ state,
+ stageUrl,
+ csrfToken,
+ (ui, idx) =>
+ addRetryButton(
+ ui,
+ idx,
+ state,
+ fileInput,
+ stageUrl,
+ csrfToken,
+ checkComplete,
+ ),
+ checkComplete,
+ );
+ });
+}
+
+(() => {
+ const config = document.getElementById("upload-config");
+ if (!config) return;
+
+ const stageUrl = config.dataset.stageUrl;
+ const finaliseUrl = config.dataset.finaliseUrl;
+ if (!stageUrl || !finaliseUrl) return;
+
+ const form = document
+ .querySelector('input[name="variant"][value="add_files"]')
+ ?.closest("form");
+ if (!form) return;
+
+ const fileInput = form.querySelector('input[type="file"]');
+ const container = document.getElementById("upload-progress-container");
+ const csrfToken = form.querySelector('input[name="csrf_token"]')?.value;
+ const state = {
+ activeUploads: new Map(),
+ completedCount: 0,
+ failedCount: 0,
+ totalFiles: 0,
+ };
+
+ const checkComplete = () =>
+ checkAllComplete(state, container, finaliseUrl, csrfToken);
+
+ form.addEventListener("submit", (e) =>
+ handleFormSubmit(
+ e,
+ form,
+ fileInput,
+ container,
+ state,
+ stageUrl,
+ csrfToken,
+ checkComplete,
+ ),
+ );
+})();
diff --git a/atr/util.py b/atr/util.py
index 0c6ebff..e3fbdf0 100644
--- a/atr/util.py
+++ b/atr/util.py
@@ -482,6 +482,12 @@ def get_unfinished_dir() -> pathlib.Path:
return pathlib.Path(config.get().UNFINISHED_STORAGE_DIR)
+def get_upload_staging_dir(session_token: str) -> pathlib.Path:
+ if not session_token.isalnum():
+ raise ValueError("Invalid session token")
+ return get_tmp_dir() / "upload-staging" / session_token
+
+
async def get_urls_as_completed(urls: Sequence[str]) -> AsyncGenerator[tuple[str, int | str | None, bytes]]:
"""GET a list of URLs in parallel and yield (url, status, content_bytes) as they become available."""
async with aiohttp.ClientSession() as session:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org