You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2025/03/20 19:10:01 UTC
(tooling-trusted-release) branch main updated: Validate editable ongoing release rsync command paths
This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
The following commit(s) were added to refs/heads/main by this push:
new 6b7c05f Validate editable ongoing release rsync command paths
6b7c05f is described below
commit 6b7c05fd8d2d4ae760f9de77539b1e67fb1a4981
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Thu Mar 20 21:09:25 2025 +0200
Validate editable ongoing release rsync command paths
Adding validation of these paths depended on further changes:
* Add project to each release in Project.editable_releases
* Fix a bug in the performance checker
* Add an editable releases method to the new custom session
* Make the number of release files function return the actual answer
* Refactor validation of rsync commands in the SSH server
* Move editable ongoing releases to the top of the add files page
---
atr/db/models.py | 12 ++--
atr/routes/__init__.py | 7 +-
atr/routes/files.py | 63 +++++++++--------
atr/ssh.py | 156 +++++++++++++++++++++++++++++++------------
atr/templates/files-add.html | 94 +++++++++++++-------------
atr/user.py | 71 ++++++++++++++++++++
6 files changed, 278 insertions(+), 125 deletions(-)
diff --git a/atr/db/models.py b/atr/db/models.py
index 9845966..09c675b 100644
--- a/atr/db/models.py
+++ b/atr/db/models.py
@@ -166,11 +166,15 @@ class Project(sqlmodel.SQLModel, table=True):
.order_by(db.validate_instrumented_attribute(Release.created).desc())
)
+ results = []
async with db.session() as data:
- results = []
- for release in (await data.execute(query)).all():
- results.append(release[0])
- return results
+ for result in (await data.execute(query)).all():
+ release = result[0]
+ results.append(release)
+ for release in results:
+ # Don't need to eager load and lose it when the session closes
+ release.project = self
+ return results
class DistributionChannel(sqlmodel.SQLModel, table=True):
diff --git a/atr/routes/__init__.py b/atr/routes/__init__.py
index 849f78f..77e7bfa 100644
--- a/atr/routes/__init__.py
+++ b/atr/routes/__init__.py
@@ -21,14 +21,14 @@ import logging
import pathlib
import time
from collections.abc import Awaitable, Callable, Coroutine
-from typing import Any, ParamSpec, TypeVar
+from typing import Any, Final, ParamSpec, TypeVar
import aiofiles
import aiofiles.os
+import asfquart
import quart
import werkzeug.datastructures as datastructures
-import asfquart
import atr.db.models as models
if asfquart.APP is ...:
@@ -40,6 +40,7 @@ T = TypeVar("T")
# TODO: Should get this from config, checking debug there
measure_performance: bool = True
+_LOGGER: Final = logging.getLogger(__name__)
# | 1 | RSA (Encrypt or Sign) [HAC] |
# | 2 | RSA Encrypt-Only [HAC] |
@@ -222,7 +223,7 @@ def app_route_performance_measure(route_path: str, http_methods: list[str] | Non
loop_time += loop_end - loop_start
# Raise exception if any
- future.result()
+ # future.result()
except StopIteration as e:
total_end = time.perf_counter()
total_time = total_end - total_start
diff --git a/atr/routes/files.py b/atr/routes/files.py
index afb97e8..9436bea 100644
--- a/atr/routes/files.py
+++ b/atr/routes/files.py
@@ -19,22 +19,27 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, Any, NoReturn, Protocol, TypeVar
+import os
+from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeVar
+import aiofiles.os
import asfquart.auth as auth
import asfquart.base as base
import asfquart.session as session
import quart
-import atr.db as db
+import atr.config as config
import atr.db.models as models
import atr.routes as routes
+import atr.user as user
if TYPE_CHECKING:
from collections.abc import Awaitable, Callable
R = TypeVar("R", covariant=True)
+_CONFIG: Final = config.get()
+
# This is the type of functions to which we apply @committer_get
# In other words, functions which accept CommitterSession as their first arg
@@ -51,6 +56,7 @@ class CommitterSession:
"""Session with extra information about committers."""
def __init__(self, web_session: session.ClientSession) -> None:
+ self._projects: list[models.Project] | None = None
self._session = web_session
def __getattr__(self, name: str) -> Any:
@@ -69,33 +75,18 @@ class CommitterSession:
return request_host
@property
- async def user_projects(self) -> list[Any]:
- user_projects: list[models.Project] = []
- async with db.session() as data:
- projects = await data.project(_committee=True, _releases=True).all()
- for p in projects:
- if p.committee is None:
- continue
- if (self.uid in p.committee.committee_members) or (self.uid in p.committee.committers):
- user_projects.append(p)
+ async def user_editable_releases(self) -> list[Any]:
+ return await user.editable_releases(self.uid, user_projects=self._projects)
- return user_projects
+ @property
+ async def user_projects(self) -> list[Any]:
+ if self._projects is None:
+ self._projects = await user.projects(self.uid)
+ return self._projects
@property
async def user_releases(self) -> list[Any]:
- user_releases: list[models.Release] = []
- async with db.session() as data:
- # TODO: We're limiting this to candidates
- # We should either call this user_candidate_releases, or change the query
- releases = await data.release(stage=models.ReleaseStage.CANDIDATE, _project=True, _committee=True).all()
- user_releases = []
- for r in releases:
- if r.committee is None:
- continue
- if (self.uid in r.committee.committee_members) or (self.uid in r.committee.committers):
- user_releases.append(r)
-
- return user_releases
+ return await user.releases(self.uid)
# This is the type of functions to which we apply @app_route
@@ -141,19 +132,33 @@ def committer_get(path: str) -> Callable[[CommitterRouteHandler[R]], RouteHandle
return decorator
-def number_of_release_files(release: models.Release) -> int:
+async def number_of_release_files(release: models.Release) -> int:
"""Return the number of files in the release."""
- # TODO: Return the number of files in the release
- return 0
+ path_project = release.project.name
+ path_version = release.version
+ path = os.path.join(_CONFIG.STATE_DIR, "rsync-files", path_project, path_version)
+ try:
+ filenames = await aiofiles.os.listdir(path)
+ except FileNotFoundError:
+ number = 0
+ else:
+ number = len(filenames)
+ return number
@committer_get("/files/add")
async def root_files_add(session: CommitterSession) -> str:
"""Show a page to allow the user to rsync files to editable releases."""
+ # Do them outside of the template rendering call to ensure order
+ # The user_editable_releases call can use cached results from user_projects
+ user_projects = await session.user_projects
+ user_editable_releases = await session.user_editable_releases
+
return await quart.render_template(
"files-add.html",
asf_id=session.uid,
- projects=await session.user_projects,
+ projects=user_projects,
server_domain=session.host,
number_of_release_files=number_of_release_files,
+ editable_releases=user_editable_releases,
)
diff --git a/atr/ssh.py b/atr/ssh.py
index a650234..ad4058f 100644
--- a/atr/ssh.py
+++ b/atr/ssh.py
@@ -21,6 +21,7 @@ import asyncio
import asyncio.subprocess
import logging
import os
+import string
import uuid
from typing import Final
@@ -30,7 +31,8 @@ import asyncssh
import atr.config as config
import atr.db as db
-from atr.db import models
+import atr.db.models as models
+import atr.user as user
_LOGGER: Final = logging.getLogger(__name__)
_CONFIG: Final = config.get()
@@ -126,43 +128,114 @@ async def server_stop(server: asyncssh.SSHAcceptor) -> None:
_LOGGER.info("SSH server stopped")
-async def _command_validate(process: asyncssh.SSHServerProcess) -> list[str] | None:
- def fail(message: str) -> list[str] | None:
- # NOTE: Changing the return type to just None really confuses mypy
- _LOGGER.error(message)
- process.exit(1)
- return None
+def _command_path_validate(path: str) -> tuple[str, str] | str:
+ if not path.startswith("/"):
+ return "The fifth argument should be an absolute path"
- command = process.command
- if not command:
- return fail("No command specified")
+ if not path.endswith("/"):
+ # Technically we could ignore this, because we rewrite the path anyway
+ # But we should enforce good rsync usage practices
+ return "The fifth argument should be a directory path, ending with a /"
- _LOGGER.info(f"Command received: {command}")
- argv = command.split()
+ if "//" in path:
+ return "The fifth argument should not contain //"
+
+ if path.count("/") != 3:
+ return "The fifth argument should be a /PROJECT/VERSION/ directory path"
+
+ path_project, path_version = path.strip("/").split("/", 1)
+ alphanum = set(string.ascii_letters + string.digits)
+ if not all(c in alphanum for c in path_project):
+ return "The project name should contain only alphanumeric characters"
+
+ # From a survey of version numbers we find that only . and - are used
+ # We also allow + which is in common use
+ version_punctuation = set(".-+")
+ if path_version[0] not in alphanum:
+ # Must certainly not allow the directory to be called "." or ".."
+ # And we also want to avoid patterns like ".htaccess"
+ return "The version should start with an alphanumeric character"
+ if path_version[-1] not in alphanum:
+ return "The version should end with an alphanumeric character"
+ if not all(c in (alphanum | version_punctuation) for c in path_version):
+ return "The version should contain only alphanumeric characters, dots, dashes, or pluses"
+
+ return path_project, path_version
+
+def _command_simple_validate(argv: list[str]) -> str | None:
if len(argv) != 5:
- return fail("There should be 5 arguments")
+ return "There should be 5 arguments"
if argv[0] != "rsync":
- return fail("The first argument should be rsync")
+ return "The first argument should be rsync"
if argv[1] != "--server":
- return fail("The second argument should be --server")
+ return "The second argument should be --server"
# TODO: Might need to accept permutations of this
# Also certain versions of rsync might change the options
acceptable_options: Final[str] = "vlogDtpre"
if not argv[2].startswith(f"-{acceptable_options}."):
- return fail(f"The third argument should start with -{acceptable_options}.")
+ return f"The third argument should start with -{acceptable_options}."
if not argv[2][len(f"-{acceptable_options}.") :].isalpha():
- return fail("The third argument should be a valid command")
+ return "The third argument should be a valid command"
if argv[3] != ".":
- return fail("The fourth argument should be .")
+ return "The fourth argument should be ."
+
+ return None
- if argv[4] != "/":
- return fail("The fifth argument should be /")
+
+async def _command_validate(process: asyncssh.SSHServerProcess) -> list[str] | None:
+ def fail(message: str) -> list[str] | None:
+ # NOTE: Changing the return type to just None really confuses mypy
+ _LOGGER.error(message)
+ process.stderr.write(f"ATR SSH error: {message}\n\n".encode())
+ process.exit(1)
+ return None
+
+ command = process.command
+ if not command:
+ return fail("No command specified")
+
+ _LOGGER.info(f"Command received: {command}")
+ argv = command.split()
+
+ error = _command_simple_validate(argv)
+ if error:
+ return fail(error)
+
+ result = _command_path_validate(argv[4])
+ if isinstance(result, str):
+ return fail(result)
+ path_project, path_version = result
+
+ # Ensure that the user has permission to upload to this project
+ async with db.session() as data:
+ project = await data.project(name=path_project, _committee=True).get()
+ if not project:
+ # Projects are public, so existence information is public
+ return fail("This project does not exist")
+ release = await data.release(project_id=project.id, version=path_version).get()
+ # The SSH UID has also been validated by SSH as being the ASF UID
+ # Since users can only set an SSH key when authenticated using ASF OAuth
+ ssh_uid = process.get_extra_info("username")
+ if not release:
+ # The user is requesting to create a new release
+ # Check if the user has permission to create a release for this project
+ if not user.is_committee_member(project.committee, ssh_uid):
+ return fail("You must be a member of this project's committee to create a release")
+ else:
+ # The user is requesting to upload to an existing release
+ # Check if the user has permission to upload to this release
+ if not user.is_committer(release.committee, ssh_uid):
+ return fail("You must be a member of this project's committee or a committer to upload to this release")
+
+ # Set the target directory to the release storage directory
+ argv[4] = os.path.join(_CONFIG.STATE_DIR, "rsync-files", path_project, path_version)
+ _LOGGER.info(f"Modified command: {argv}")
return argv
@@ -176,13 +249,8 @@ async def _handle_client(process: asyncssh.SSHServerProcess) -> None:
if not argv:
return
- # Create a storage directory for this user if it doesn't exist
- user_storage_dir = os.path.join(_CONFIG.STATE_DIR, "rsync-files", asf_uid)
- await aiofiles.os.makedirs(user_storage_dir, exist_ok=True)
-
- # Set the target directory to the user's storage directory
- argv[4] = user_storage_dir
- _LOGGER.info(f"Modified command: {argv}")
+ # Create the release's storage directory if it doesn't exist
+ await aiofiles.os.makedirs(argv[4], exist_ok=True)
try:
# Create subprocess to actually run the command
@@ -200,23 +268,27 @@ async def _handle_client(process: asyncssh.SSHServerProcess) -> None:
# Wait for the process to complete
exit_status = await proc.wait()
+
# Start a task to process the new files
- async with db.session() as data:
- async with data.begin():
- from atr.tasks.rsync import Analyse
-
- upload_uuid = str(uuid.uuid4()).replace("-", "")
- project_name = "test"
- data.add(
- models.Task(
- status=models.TaskStatus.QUEUED,
- task_type="rsync_analyse",
- task_args=Analyse(
- asf_uid=asf_uid, upload_uuid=upload_uuid, project_name=project_name
- ).model_dump(),
+ # TODO: Turned this off for now, because we need to decide on the workflow
+ start_task = False
+ if start_task:
+ async with db.session() as data:
+ async with data.begin():
+ from atr.tasks.rsync import Analyse
+
+ upload_uuid = str(uuid.uuid4()).replace("-", "")
+ project_name = "test"
+ data.add(
+ models.Task(
+ status=models.TaskStatus.QUEUED,
+ task_type="rsync_analyse",
+ task_args=Analyse(
+ asf_uid=asf_uid, upload_uuid=upload_uuid, project_name=project_name
+ ).model_dump(),
+ )
)
- )
- # Exit the SSH process with the same status
+ # Exit the SSH process with the same status as the rsync process
process.exit(exit_status)
except Exception as e:
diff --git a/atr/templates/files-add.html b/atr/templates/files-add.html
index f2e77f3..95891d8 100644
--- a/atr/templates/files-add.html
+++ b/atr/templates/files-add.html
@@ -25,6 +25,42 @@
</ul>
</p>
+ <h2>Editable ongoing releases</h2>
+ <div class="row row-cols-1 row-cols-md-2 g-4 mb-5">
+ {% for release in editable_releases %}
+ <div class="col" id="{{ release.name }}">
+ <div class="card h-100">
+ <div class="card-body position-relative">
+ <div class="position-absolute top-0 end-0 m-2">
+ <span class="badge bg-success">Ongoing</span>
+ </div>
+ <h5 class="card-title">{{ release.project.display_name }} {{ release.version }}</h5>
+ {% if release.project.committee %}
+ <h6 class="card-subtitle mb-2 text-muted">{{ release.project.committee.display_name }}</h6>
+ {% endif %}
+ <p class="card-text">
+ {% if number_of_release_files(release) > 0 %}
+ This editable ongoing release has {{ number_of_release_files(release) }} file(s).
+ {% else %}
+ This editable ongoing release doesn't have any files yet.
+ {% endif %}
+ Use the command below to add or modify files in this release:
+ </p>
+ </div>
+ <pre class="card-footer bg-light border-1 pt-4 small">
+rsync -av -e 'ssh -p 2222' your/files/ \
+ {{ asf_id }}@{{ server_domain }}:/{{ release.project.name }}/{{ release.version }}/
+ </pre>
+ </div>
+ </div>
+ {% endfor %}
+ {% if editable_releases|length == 0 %}
+ <div class="col-12">
+ <div class="alert alert-info">There are currently no editable ongoing releases.</div>
+ </div>
+ {% endif %}
+ </div>
+
<h2>Projects</h2>
<div class="row row-cols-1 row-cols-md-2 g-4 mb-5">
{% for project in projects %}
@@ -44,16 +80,18 @@
{% endif %}
<p class="card-text">
{% if editable_releases|length > 0 %}
- This project already has ongoing releases:
- {% for release in editable_releases %}<a href="#{{ release.name }}">{{ release.version }}</a>;{% endfor %}
- to create another one, use the command below.
+ This project already has the ongoing releases
+ {% for release in editable_releases %}
+ <code><a href="#{{ release.name }}" class="border rounded px-2">{{ release.version }}</a></code>,
+ {% endfor %}
+ but to create another one, use the command below.
{% else %}
This project does not have an editable ongoing release.
To create one and add files, use the command below.
{% endif %}
</p>
</div>
- <pre class="card-footer bg-transparent border-top-0 small">
+ <pre class="card-footer bg-light border-1 pt-4 small">
rsync -av -e 'ssh -p 2222' your/files/ \
{{ asf_id }}@{{ server_domain }}:/{{ project.name }}/VERSION/
</pre>
@@ -72,9 +110,11 @@ rsync -av -e 'ssh -p 2222' your/files/ \
{% endif %}
<p class="card-text">
{% if editable_releases|length > 0 %}
- This project already has ongoing releases:
- {% for release in editable_releases %}<a href="#{{ release.name }}">{{ release.version }}</a>;{% endfor %}
- to create another one, you must be a member of the project's committee.
+ This project already has the ongoing releases
+ {% for release in editable_releases %}
+ <code><a href="#{{ release.name }}" class="border rounded px-2">{{ release.version }}</a></code>,
+ {% endfor %}
+ but to create another one, use the command below.
{% else %}
This project does not have an editable ongoing release.
To create one, you must be a member of the project's committee.
@@ -86,44 +126,4 @@ rsync -av -e 'ssh -p 2222' your/files/ \
{% endif %}
{% endfor %}
</div>
-
- <h2>Editable ongoing releases</h2>
- <div class="row row-cols-1 row-cols-md-2 g-4 mb-5">
- {% set ns = namespace(has_releases=false) %}
- {% for project in projects %}
- {% for release in project.editable_releases %}
- {% set ns.has_releases = true %}
- <div class="col" id="{{ release.name }}">
- <div class="card h-100">
- <div class="card-body position-relative">
- <div class="position-absolute top-0 end-0 m-2">
- <span class="badge bg-success">Ongoing</span>
- </div>
- <h5 class="card-title">{{ project.display_name }} {{ release.version or "?" }}</h5>
- {% if project.committee %}
- <h6 class="card-subtitle mb-2 text-muted">{{ project.committee.display_name }}</h6>
- {% endif %}
- <p class="card-text">
- {% if number_of_release_files(release) > 0 %}
- This editable ongoing release has {{ number_of_release_files(release) }} file(s).
- {% else %}
- This editable ongoing release doesn't have any files yet.
- {% endif %}
- Use the command below to add or modify files in this release:
- </p>
- </div>
- <pre class="card-footer bg-transparent border-top-0 small">
-rsync -av -e 'ssh -p 2222' your/files/ \
- {{ asf_id }}@{{ server_domain }}:/{{ project.name }}/{{ release.version }}/
- </pre>
- </div>
- </div>
- {% endfor %}
- {% endfor %}
- {% if not ns.has_releases %}
- <div class="col-12">
- <div class="alert alert-info">There are currently no editable ongoing releases.</div>
- </div>
- {% endif %}
- </div>
{% endblock content %}
diff --git a/atr/user.py b/atr/user.py
new file mode 100644
index 0000000..9b0f7a7
--- /dev/null
+++ b/atr/user.py
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""user.py"""
+
+import atr.db as db
+import atr.db.models as models
+
+
+async def editable_releases(uid: str, user_projects: list[models.Project] | None = None) -> list[models.Release]:
+ if user_projects is None:
+ user_projects = await projects(uid)
+ user_editable_releases: list[models.Release] = []
+ for p in user_projects:
+ releases = await p.editable_releases
+ user_editable_releases.extend(releases)
+ return user_editable_releases
+
+
+def is_committer(committee: models.Committee | None, uid: str) -> bool:
+ if committee is None:
+ return False
+ return any((committer_uid == uid) for committer_uid in committee.committers)
+
+
+def is_committee_member(committee: models.Committee | None, uid: str) -> bool:
+ if committee is None:
+ return False
+ return any((member_uid == uid) for member_uid in committee.committee_members)
+
+
+async def projects(uid: str) -> list[models.Project]:
+ user_projects: list[models.Project] = []
+ async with db.session() as data:
+ # Must have releases, because this is used in editable_releases
+ projects = await data.project(_committee=True, _releases=True).all()
+ for p in projects:
+ if p.committee is None:
+ continue
+ if (uid in p.committee.committee_members) or (uid in p.committee.committers):
+ user_projects.append(p)
+ return user_projects
+
+
+async def releases(uid: str) -> list[models.Release]:
+ user_releases: list[models.Release] = []
+ async with db.session() as data:
+ # TODO: We're limiting this to candidates
+ # We should either call this user_candidate_releases, or change the query
+ releases = await data.release(stage=models.ReleaseStage.CANDIDATE, _project=True, _committee=True).all()
+ user_releases = []
+ for r in releases:
+ if r.committee is None:
+ continue
+ if (uid in r.committee.committee_members) or (uid in r.committee.committers):
+ user_releases.append(r)
+ return user_releases
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org