You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2025/04/15 19:43:54 UTC
(tooling-trusted-release) branch main updated: Add the ability to import files from a remote SVN repository into a draft
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
The following commit(s) were added to refs/heads/main by this push:
new c7b08a4 Add the ability to import files from a remote SVN repository into a draft
c7b08a4 is described below
commit c7b08a4e7e09bea0aac97b50bfa21938a8db55ea
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Tue Apr 15 20:43:22 2025 +0100
Add the ability to import files from a remote SVN repository into a draft
---
atr/db/models.py | 1 +
atr/routes/draft.py | 81 ++++++++++++++++++++
atr/tasks/__init__.py | 3 +
atr/tasks/svn.py | 162 +++++++++++++++++++++++++++++++++++++++
atr/templates/draft-svnload.html | 69 +++++++++++++++++
atr/templates/drafts.html | 3 +
6 files changed, 319 insertions(+)
diff --git a/atr/db/models.py b/atr/db/models.py
index 9a1b3a3..2d0b680 100644
--- a/atr/db/models.py
+++ b/atr/db/models.py
@@ -297,6 +297,7 @@ class TaskType(str, enum.Enum):
# RSYNC_ANALYSE = "rsync_analyse"
SBOM_GENERATE_CYCLONEDX = "sbom_generate_cyclonedx"
SIGNATURE_CHECK = "signature_check"
+ SVN_IMPORT_FILES = "svn_import_files"
TARGZ_INTEGRITY = "targz_integrity"
TARGZ_STRUCTURE = "targz_structure"
VOTE_INITIATE = "vote_initiate"
diff --git a/atr/routes/draft.py b/atr/routes/draft.py
index 97471b5..7a53c55 100644
--- a/atr/routes/draft.py
+++ b/atr/routes/draft.py
@@ -110,6 +110,31 @@ class PromoteForm(util.QuartFormTyped):
submit = wtforms.SubmitField("Promote candidate draft")
+class SvnImportForm(util.QuartFormTyped):
+ """Form for importing files from SVN into a draft."""
+
+ svn_url = wtforms.URLField(
+ "SVN URL",
+ validators=[
+ wtforms.validators.InputRequired("SVN URL is required."),
+ wtforms.validators.URL(require_tld=False),
+ ],
+ description="The URL to the public SVN directory",
+ )
+ revision = wtforms.StringField(
+ "Revision",
+ default="HEAD",
+ validators=[],
+ description="Specify an SVN revision number or leave as HEAD for the latest",
+ )
+ target_subdirectory = wtforms.StringField(
+ "Target subdirectory",
+ validators=[],
+ description="Subdirectory to place imported files, defaulting to the root (optional)",
+ )
+ submit = wtforms.SubmitField("Queue SVN import task")
+
+
@routes.committer("/draft/add", methods=["GET", "POST"])
async def add(session: routes.CommitterSession) -> response.Response | str:
"""Show a page to allow the user to rsync files to candidate drafts."""
@@ -801,6 +826,62 @@ async def sbomgen(
)
+@routes.committer("/draft/svnload/<project_name>/<version_name>", methods=["GET", "POST"])
+async def svnload(session: routes.CommitterSession, project_name: str, version_name: str) -> response.Response | str:
+ """Import files from SVN into a draft."""
+
+ form = await SvnImportForm.create_form()
+
+ if not any((p.name == project_name) for p in (await session.user_projects)):
+ raise base.ASFQuartException("You do not have access to this project", errorcode=403)
+
+ async with db.session() as data:
+ release = await data.release(name=models.release_name(project_name, version_name)).demand(
+ base.ASFQuartException("Release does not exist", errorcode=404)
+ )
+ if release.phase != models.ReleasePhase.RELEASE_CANDIDATE_DRAFT:
+ raise base.ASFQuartException("SVN import is only available for candidate drafts", errorcode=400)
+
+ if await form.validate_on_submit():
+ try:
+ task_args = {
+ "svn_url": str(form.svn_url.data),
+ "revision": str(form.revision.data),
+ "target_subdirectory": str(form.target_subdirectory.data) if form.target_subdirectory.data else None,
+ "project_name": project_name,
+ "version_name": version_name,
+ "asf_uid": session.uid,
+ }
+ async with db.session() as data:
+ svn_import_task = models.Task(
+ task_type=models.TaskType.SVN_IMPORT_FILES,
+ task_args=task_args,
+ added=datetime.datetime.now(datetime.UTC),
+ status=models.TaskStatus.QUEUED,
+ release_name=release.name,
+ )
+ data.add(svn_import_task)
+ await data.commit()
+
+ return await session.redirect(
+ evaluate,
+ success="SVN import task queued successfully",
+ project_name=project_name,
+ version_name=version_name,
+ )
+ except Exception as e:
+ logging.exception("Error queueing SVN import task:")
+ await quart.flash(f"Error queueing SVN import task: {e!s}", "error")
+
+ return await quart.render_template(
+ "draft-svnload.html",
+ asf_id=session.uid,
+ project_name=project_name,
+ version_name=version_name,
+ form=form,
+ )
+
+
@routes.committer("/draft/tools/<project_name>/<version_name>/<path:file_path>")
async def tools(session: routes.CommitterSession, project_name: str, version_name: str, file_path: str) -> str:
"""Show the tools for a specific file."""
diff --git a/atr/tasks/__init__.py b/atr/tasks/__init__.py
index 8f49685..61e86d0 100644
--- a/atr/tasks/__init__.py
+++ b/atr/tasks/__init__.py
@@ -31,6 +31,7 @@ import atr.tasks.checks.zipformat as zipformat
# import atr.tasks.rsync as rsync
import atr.tasks.sbom as sbom
+import atr.tasks.svn as svn
import atr.tasks.vote as vote
import atr.util as util
@@ -118,6 +119,8 @@ def resolve(task_type: models.TaskType) -> Callable[..., Awaitable[str | None]]:
return sbom.generate_cyclonedx
case models.TaskType.SIGNATURE_CHECK:
return signature.check
+ case models.TaskType.SVN_IMPORT_FILES:
+ return svn.import_files
case models.TaskType.TARGZ_INTEGRITY:
return targz.integrity
case models.TaskType.TARGZ_STRUCTURE:
diff --git a/atr/tasks/svn.py b/atr/tasks/svn.py
new file mode 100644
index 0000000..23ede1b
--- /dev/null
+++ b/atr/tasks/svn.py
@@ -0,0 +1,162 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import asyncio
+import logging
+import pathlib
+from typing import Any, Final
+
+import aiofiles.os
+import aioshutil
+import pydantic
+
+import atr.revision as revision
+import atr.tasks.checks as checks
+
+_LOGGER: Final = logging.getLogger(__name__)
+
+
+class SvnImport(pydantic.BaseModel):
+ """Arguments for the task to import files from SVN."""
+
+ svn_url: str
+ revision: str
+ target_subdirectory: str | None
+ project_name: str
+ version_name: str
+ asf_uid: str
+
+
+class SvnImportError(Exception):
+ """Custom exception for SVN import failures."""
+
+ def __init__(self, message: str, details: dict[str, Any] | None = None) -> None:
+ super().__init__(message)
+ self.details = details or {}
+
+
+@checks.with_model(SvnImport)
+async def import_files(args: SvnImport) -> str | None:
+ """Import files from SVN into a draft release candidate revision."""
+ try:
+ result_message = await _import_files_core(args)
+ return result_message
+ except SvnImportError as e:
+ _LOGGER.error(f"SVN import failed: {e.details}")
+ raise
+ except Exception:
+ _LOGGER.exception("Unexpected error during SVN import task")
+ raise
+
+
+async def _import_files_core(args: SvnImport) -> str:
+ """Core logic to perform the SVN export."""
+ _LOGGER.info(f"Starting SVN import for {args.project_name}-{args.version_name}")
+ # We have to use a temporary directory otherwise SVN thinks it's a pegged revision
+ temp_export_dir_name = ".svn-export.tmp"
+
+ async with revision.create_and_manage(args.project_name, args.version_name, args.asf_uid) as (
+ new_revision_dir,
+ new_revision_name,
+ ):
+ _LOGGER.debug(f"Created revision directory: {new_revision_dir}")
+
+ final_target_path = new_revision_dir
+ if args.target_subdirectory:
+ final_target_path = new_revision_dir / args.target_subdirectory
+ # Validate that final_target_path is a subdirectory of new_revision_dir
+ if not final_target_path.is_relative_to(new_revision_dir):
+ raise SvnImportError(
+ f"Target subdirectory {args.target_subdirectory} is not a subdirectory of {new_revision_dir}"
+ )
+ await aiofiles.os.makedirs(final_target_path, exist_ok=True)
+
+ temp_export_path = new_revision_dir / temp_export_dir_name
+
+ svn_command = [
+ "svn",
+ "export",
+ "--non-interactive",
+ "--trust-server-cert-failures",
+ "unknown-ca,cn-mismatch",
+ "-r",
+ args.revision,
+ "--",
+ args.svn_url,
+ str(temp_export_path),
+ ]
+
+ await _import_files_core_run_svn_export(svn_command, temp_export_path)
+
+ # Move files from temp export path to final target path
+ # We only have to do this to avoid the SVN pegged revision issue
+ _LOGGER.info(f"Moving exported files from {temp_export_path} to {final_target_path}")
+ for item_name in await aiofiles.os.listdir(temp_export_path):
+ source_item = temp_export_path / item_name
+ destination_item = final_target_path / item_name
+ try:
+ await aioshutil.move(str(source_item), str(destination_item))
+ except FileExistsError:
+ _LOGGER.warning(f"Item {destination_item} already exists, skipping move for {item_name}")
+ except Exception as move_err:
+ _LOGGER.error(f"Error moving {source_item} to {destination_item}: {move_err}")
+ await aiofiles.os.rmdir(temp_export_path)
+ _LOGGER.info(f"Removed temporary export directory: {temp_export_path}")
+
+ return f"Successfully imported files from SVN into revision {new_revision_name}"
+
+
+async def _import_files_core_run_svn_export(svn_command: list[str], temp_export_path: pathlib.Path) -> None:
+ """Execute the svn export command and handle errors."""
+ _LOGGER.info(f"Executing SVN command: {' '.join(svn_command)}")
+
+ timeout_seconds = 600
+ try:
+ process = await asyncio.create_subprocess_exec(
+ *svn_command,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout_seconds)
+
+ stdout_str = stdout.decode("utf-8", errors="ignore").strip() if stdout else ""
+ stderr_str = stderr.decode("utf-8", errors="ignore").strip() if stderr else ""
+
+ if process.returncode != 0:
+ _LOGGER.error(f"SVN export failed with code {process.returncode}")
+ _LOGGER.error(f"SVN stderr: {stderr_str}")
+ _LOGGER.error(f"SVN stdout: {stdout_str[:1000]}")
+ raise SvnImportError(
+ f"SVN export failed with code {process.returncode}",
+ details={"returncode": process.returncode, "stderr": stderr_str, "stdout": stdout_str[:1000]},
+ )
+
+ _LOGGER.info("SVN export to temporary directory successful")
+ if stdout_str:
+ _LOGGER.debug(f"SVN stdout: {stdout_str}")
+ if stderr_str:
+ _LOGGER.warning(f"SVN stderr: {stderr_str}")
+
+ except TimeoutError:
+ _LOGGER.error("SVN export command timed out after %d seconds", timeout_seconds)
+ raise SvnImportError("SVN export command timed out")
+ except FileNotFoundError:
+ _LOGGER.error("svn command not found. Is it installed and in PATH?")
+ raise SvnImportError("svn command not found")
+ except Exception as e:
+ _LOGGER.exception("Unexpected error during SVN export subprocess execution")
+ raise SvnImportError(f"Unexpected error during SVN export: {e}")
diff --git a/atr/templates/draft-svnload.html b/atr/templates/draft-svnload.html
new file mode 100644
index 0000000..746cfa5
--- /dev/null
+++ b/atr/templates/draft-svnload.html
@@ -0,0 +1,69 @@
+{% extends "layouts/base.html" %}
+
+{% block title %}
+ Import from SVN into {{ project_name }}-{{ version_name }} ~ ATR
+{% endblock title %}
+
+{% block description %}
+ Import files from a Subversion repository into a candidate draft.
+{% endblock description %}
+
+{% block content %}
+ <p>
+ <a href="{{ as_url(routes.draft.evaluate, project_name=project_name, version_name=version_name) }}"
+ class="back-link">← Back to Evaluate files</a>
+ </p>
+ <h1>Import from SVN into {{ project_name }}-{{ version_name }}</h1>
+ <p class="intro">Import files from a publicly readable Subversion repository URL into this draft.</p>
+ <p>
+ The import will be processed in the background using the <code>svn export</code> command.
+ You can monitor progress on the <em>Evaluate files</em> page for this draft once the task is queued.
+ </p>
+
+ <div class="row">
+ <div class="col-md-8 w-100">
+ <form method="post" novalidate class="atr-canary py-4 px-5">
+ {{ form.hidden_tag() }}
+ <div class="mb-3 pb-3 row border-bottom">
+ <label for="{{ form.svn_url.id }}"
+ class="col-sm-3 col-form-label text-sm-end">{{ form.svn_url.label.text }}:</label>
+ <div class="col-sm-9">
+ {{ form.svn_url(class_="form-control") }}
+ {% if form.svn_url.description %}<small class="form-text text-muted">{{ form.svn_url.description }}</small>{% endif %}
+ {% if form.svn_url.errors -%}<div class="invalid-feedback d-block">{{ form.svn_url.errors[0] }}</div>{%- endif %}
+ </div>
+ </div>
+
+ <div class="mb-3 pb-3 row border-bottom">
+ <label for="{{ form.revision.id }}"
+ class="col-sm-3 col-form-label text-sm-end">{{ form.revision.label.text }}:</label>
+ <div class="col-sm-9">
+ {{ form.revision(class_="form-control") }}
+ {% if form.revision.description %}
+ <small class="form-text text-muted">{{ form.revision.description }}</small>
+ {% endif %}
+ {% if form.revision.errors -%}<div class="invalid-feedback d-block">{{ form.revision.errors[0] }}</div>{%- endif %}
+ </div>
+ </div>
+
+ <div class="mb-3 pb-3 row border-bottom">
+ <label for="{{ form.target_subdirectory.id }}"
+ class="col-sm-3 col-form-label text-sm-end">{{ form.target_subdirectory.label.text }}:</label>
+ <div class="col-sm-9">
+ {{ form.target_subdirectory(class_="form-control") }}
+ {% if form.target_subdirectory.description %}
+ <small class="form-text text-muted">{{ form.target_subdirectory.description }}</small>
+ {% endif %}
+ {% if form.target_subdirectory.errors -%}
+ <div class="invalid-feedback d-block">{{ form.target_subdirectory.errors[0] }}</div>
+ {%- endif %}
+ </div>
+ </div>
+
+ <div class="row">
+ <div class="col-sm-9 offset-sm-3">{{ form.submit(class_="btn btn-primary mt-3") }}</div>
+ </div>
+ </form>
+ </div>
+ </div>
+{% endblock content %}
diff --git a/atr/templates/drafts.html b/atr/templates/drafts.html
index be9ce8f..72d212c 100644
--- a/atr/templates/drafts.html
+++ b/atr/templates/drafts.html
@@ -51,6 +51,9 @@
<a href="{{ as_url(routes.draft.add_file, project_name=release.project.name, version_name=release.version) }}"
title="Upload file for {{ release.project.display_name }} {{ release.version }}"
class="btn btn-sm btn-outline-secondary">Upload file</a>
+ <a href="{{ as_url(routes.draft.svnload, project_name=release.project.name, version_name=release.version) }}"
+ title="Import from SVN for {{ release.project.display_name }} {{ release.version }}"
+ class="btn btn-sm btn-outline-secondary">Import from SVN</a>
<a href="{{ as_url(routes.draft.revisions, project_name=release.project.name, version_name=release.version) }}"
title="Revisions for {{ release.project.display_name }} {{ release.version }}"
class="btn btn-sm btn-outline-secondary">Revisions</a>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org