You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by sb...@apache.org on 2025/03/20 19:10:01 UTC

(tooling-trusted-release) branch main updated: Validate editable ongoing release rsync command paths

This is an automated email from the ASF dual-hosted git repository.

sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git


The following commit(s) were added to refs/heads/main by this push:
     new 6b7c05f  Validate editable ongoing release rsync command paths
6b7c05f is described below

commit 6b7c05fd8d2d4ae760f9de77539b1e67fb1a4981
Author: Sean B. Palmer <se...@miscoranda.com>
AuthorDate: Thu Mar 20 21:09:25 2025 +0200

    Validate editable ongoing release rsync command paths
    
    Adding validation of these paths depended on further changes:
    
    * Add project to each release in Project.editable_releases
    * Fix a bug in the performance checker
    * Add an editable releases method to the new custom session
    * Make the number of release files function return the actual answer
    * Refactor validation of rsync commands in the SSH server
    * Move editable ongoing releases to the top of the add files page
---
 atr/db/models.py             |  12 ++--
 atr/routes/__init__.py       |   7 +-
 atr/routes/files.py          |  63 +++++++++--------
 atr/ssh.py                   | 156 +++++++++++++++++++++++++++++++------------
 atr/templates/files-add.html |  94 +++++++++++++-------------
 atr/user.py                  |  71 ++++++++++++++++++++
 6 files changed, 278 insertions(+), 125 deletions(-)

diff --git a/atr/db/models.py b/atr/db/models.py
index 9845966..09c675b 100644
--- a/atr/db/models.py
+++ b/atr/db/models.py
@@ -166,11 +166,15 @@ class Project(sqlmodel.SQLModel, table=True):
             .order_by(db.validate_instrumented_attribute(Release.created).desc())
         )
 
+        results = []
         async with db.session() as data:
-            results = []
-            for release in (await data.execute(query)).all():
-                results.append(release[0])
-            return results
+            for result in (await data.execute(query)).all():
+                release = result[0]
+                results.append(release)
+        for release in results:
+            # Don't need to eager load and lose it when the session closes
+            release.project = self
+        return results
 
 
 class DistributionChannel(sqlmodel.SQLModel, table=True):
diff --git a/atr/routes/__init__.py b/atr/routes/__init__.py
index 849f78f..77e7bfa 100644
--- a/atr/routes/__init__.py
+++ b/atr/routes/__init__.py
@@ -21,14 +21,14 @@ import logging
 import pathlib
 import time
 from collections.abc import Awaitable, Callable, Coroutine
-from typing import Any, ParamSpec, TypeVar
+from typing import Any, Final, ParamSpec, TypeVar
 
 import aiofiles
 import aiofiles.os
+import asfquart
 import quart
 import werkzeug.datastructures as datastructures
 
-import asfquart
 import atr.db.models as models
 
 if asfquart.APP is ...:
@@ -40,6 +40,7 @@ T = TypeVar("T")
 # TODO: Should get this from config, checking debug there
 measure_performance: bool = True
 
+_LOGGER: Final = logging.getLogger(__name__)
 
 # |         1 | RSA (Encrypt or Sign) [HAC]                        |
 # |         2 | RSA Encrypt-Only [HAC]                             |
@@ -222,7 +223,7 @@ def app_route_performance_measure(route_path: str, http_methods: list[str] | Non
                     loop_time += loop_end - loop_start
 
                     # Raise exception if any
-                    future.result()
+                    # future.result()
             except StopIteration as e:
                 total_end = time.perf_counter()
                 total_time = total_end - total_start
diff --git a/atr/routes/files.py b/atr/routes/files.py
index afb97e8..9436bea 100644
--- a/atr/routes/files.py
+++ b/atr/routes/files.py
@@ -19,22 +19,27 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, NoReturn, Protocol, TypeVar
+import os
+from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeVar
 
+import aiofiles.os
 import asfquart.auth as auth
 import asfquart.base as base
 import asfquart.session as session
 import quart
 
-import atr.db as db
+import atr.config as config
 import atr.db.models as models
 import atr.routes as routes
+import atr.user as user
 
 if TYPE_CHECKING:
     from collections.abc import Awaitable, Callable
 
 R = TypeVar("R", covariant=True)
 
+_CONFIG: Final = config.get()
+
 
 # This is the type of functions to which we apply @committer_get
 # In other words, functions which accept CommitterSession as their first arg
@@ -51,6 +56,7 @@ class CommitterSession:
     """Session with extra information about committers."""
 
     def __init__(self, web_session: session.ClientSession) -> None:
+        self._projects: list[models.Project] | None = None
         self._session = web_session
 
     def __getattr__(self, name: str) -> Any:
@@ -69,33 +75,18 @@ class CommitterSession:
         return request_host
 
     @property
-    async def user_projects(self) -> list[Any]:
-        user_projects: list[models.Project] = []
-        async with db.session() as data:
-            projects = await data.project(_committee=True, _releases=True).all()
-            for p in projects:
-                if p.committee is None:
-                    continue
-                if (self.uid in p.committee.committee_members) or (self.uid in p.committee.committers):
-                    user_projects.append(p)
+    async def user_editable_releases(self) -> list[Any]:
+        return await user.editable_releases(self.uid, user_projects=self._projects)
 
-        return user_projects
+    @property
+    async def user_projects(self) -> list[Any]:
+        if self._projects is None:
+            self._projects = await user.projects(self.uid)
+        return self._projects
 
     @property
     async def user_releases(self) -> list[Any]:
-        user_releases: list[models.Release] = []
-        async with db.session() as data:
-            # TODO: We're limiting this to candidates
-            # We should either call this user_candidate_releases, or change the query
-            releases = await data.release(stage=models.ReleaseStage.CANDIDATE, _project=True, _committee=True).all()
-            user_releases = []
-            for r in releases:
-                if r.committee is None:
-                    continue
-                if (self.uid in r.committee.committee_members) or (self.uid in r.committee.committers):
-                    user_releases.append(r)
-
-        return user_releases
+        return await user.releases(self.uid)
 
 
 # This is the type of functions to which we apply @app_route
@@ -141,19 +132,33 @@ def committer_get(path: str) -> Callable[[CommitterRouteHandler[R]], RouteHandle
     return decorator
 
 
-def number_of_release_files(release: models.Release) -> int:
+async def number_of_release_files(release: models.Release) -> int:
     """Return the number of files in the release."""
-    # TODO: Return the number of files in the release
-    return 0
+    path_project = release.project.name
+    path_version = release.version
+    path = os.path.join(_CONFIG.STATE_DIR, "rsync-files", path_project, path_version)
+    try:
+        filenames = await aiofiles.os.listdir(path)
+    except FileNotFoundError:
+        number = 0
+    else:
+        number = len(filenames)
+    return number
 
 
 @committer_get("/files/add")
 async def root_files_add(session: CommitterSession) -> str:
     """Show a page to allow the user to rsync files to editable releases."""
+    # Do them outside of the template rendering call to ensure order
+    # The user_editable_releases call can use cached results from user_projects
+    user_projects = await session.user_projects
+    user_editable_releases = await session.user_editable_releases
+
     return await quart.render_template(
         "files-add.html",
         asf_id=session.uid,
-        projects=await session.user_projects,
+        projects=user_projects,
         server_domain=session.host,
         number_of_release_files=number_of_release_files,
+        editable_releases=user_editable_releases,
     )
diff --git a/atr/ssh.py b/atr/ssh.py
index a650234..ad4058f 100644
--- a/atr/ssh.py
+++ b/atr/ssh.py
@@ -21,6 +21,7 @@ import asyncio
 import asyncio.subprocess
 import logging
 import os
+import string
 import uuid
 from typing import Final
 
@@ -30,7 +31,8 @@ import asyncssh
 
 import atr.config as config
 import atr.db as db
-from atr.db import models
+import atr.db.models as models
+import atr.user as user
 
 _LOGGER: Final = logging.getLogger(__name__)
 _CONFIG: Final = config.get()
@@ -126,43 +128,114 @@ async def server_stop(server: asyncssh.SSHAcceptor) -> None:
     _LOGGER.info("SSH server stopped")
 
 
-async def _command_validate(process: asyncssh.SSHServerProcess) -> list[str] | None:
-    def fail(message: str) -> list[str] | None:
-        # NOTE: Changing the return type to just None really confuses mypy
-        _LOGGER.error(message)
-        process.exit(1)
-        return None
+def _command_path_validate(path: str) -> tuple[str, str] | str:
+    if not path.startswith("/"):
+        return "The fifth argument should be an absolute path"
 
-    command = process.command
-    if not command:
-        return fail("No command specified")
+    if not path.endswith("/"):
+        # Technically we could ignore this, because we rewrite the path anyway
+        # But we should enforce good rsync usage practices
+        return "The fifth argument should be a directory path, ending with a /"
 
-    _LOGGER.info(f"Command received: {command}")
-    argv = command.split()
+    if "//" in path:
+        return "The fifth argument should not contain //"
+
+    if path.count("/") != 3:
+        return "The fifth argument should be a /PROJECT/VERSION/ directory path"
+
+    path_project, path_version = path.strip("/").split("/", 1)
+    alphanum = set(string.ascii_letters + string.digits)
+    if not all(c in alphanum for c in path_project):
+        return "The project name should contain only alphanumeric characters"
+
+    # From a survey of version numbers we find that only . and - are used
+    # We also allow + which is in common use
+    version_punctuation = set(".-+")
+    if path_version[0] not in alphanum:
+        # Must certainly not allow the directory to be called "." or ".."
+        # And we also want to avoid patterns like ".htaccess"
+        return "The version should start with an alphanumeric character"
+    if path_version[-1] not in alphanum:
+        return "The version should end with an alphanumeric character"
+    if not all(c in (alphanum | version_punctuation) for c in path_version):
+        return "The version should contain only alphanumeric characters, dots, dashes, or pluses"
+
+    return path_project, path_version
 
+
+def _command_simple_validate(argv: list[str]) -> str | None:
     if len(argv) != 5:
-        return fail("There should be 5 arguments")
+        return "There should be 5 arguments"
 
     if argv[0] != "rsync":
-        return fail("The first argument should be rsync")
+        return "The first argument should be rsync"
 
     if argv[1] != "--server":
-        return fail("The second argument should be --server")
+        return "The second argument should be --server"
 
     # TODO: Might need to accept permutations of this
     # Also certain versions of rsync might change the options
     acceptable_options: Final[str] = "vlogDtpre"
     if not argv[2].startswith(f"-{acceptable_options}."):
-        return fail(f"The third argument should start with -{acceptable_options}.")
+        return f"The third argument should start with -{acceptable_options}."
 
     if not argv[2][len(f"-{acceptable_options}.") :].isalpha():
-        return fail("The third argument should be a valid command")
+        return "The third argument should be a valid command"
 
     if argv[3] != ".":
-        return fail("The fourth argument should be .")
+        return "The fourth argument should be ."
+
+    return None
 
-    if argv[4] != "/":
-        return fail("The fifth argument should be /")
+
+async def _command_validate(process: asyncssh.SSHServerProcess) -> list[str] | None:
+    def fail(message: str) -> list[str] | None:
+        # NOTE: Changing the return type to just None really confuses mypy
+        _LOGGER.error(message)
+        process.stderr.write(f"ATR SSH error: {message}\n\n".encode())
+        process.exit(1)
+        return None
+
+    command = process.command
+    if not command:
+        return fail("No command specified")
+
+    _LOGGER.info(f"Command received: {command}")
+    argv = command.split()
+
+    error = _command_simple_validate(argv)
+    if error:
+        return fail(error)
+
+    result = _command_path_validate(argv[4])
+    if isinstance(result, str):
+        return fail(result)
+    path_project, path_version = result
+
+    # Ensure that the user has permission to upload to this project
+    async with db.session() as data:
+        project = await data.project(name=path_project, _committee=True).get()
+        if not project:
+            # Projects are public, so existence information is public
+            return fail("This project does not exist")
+        release = await data.release(project_id=project.id, version=path_version).get()
+        # The SSH UID has also been validated by SSH as being the ASF UID
+        # Since users can only set an SSH key when authenticated using ASF OAuth
+        ssh_uid = process.get_extra_info("username")
+        if not release:
+            # The user is requesting to create a new release
+            # Check if the user has permission to create a release for this project
+            if not user.is_committee_member(project.committee, ssh_uid):
+                return fail("You must be a member of this project's committee to create a release")
+        else:
+            # The user is requesting to upload to an existing release
+            # Check if the user has permission to upload to this release
+            if not user.is_committer(release.committee, ssh_uid):
+                return fail("You must be a member of this project's committee or a committer to upload to this release")
+
+    # Set the target directory to the release storage directory
+    argv[4] = os.path.join(_CONFIG.STATE_DIR, "rsync-files", path_project, path_version)
+    _LOGGER.info(f"Modified command: {argv}")
 
     return argv
 
@@ -176,13 +249,8 @@ async def _handle_client(process: asyncssh.SSHServerProcess) -> None:
     if not argv:
         return
 
-    # Create a storage directory for this user if it doesn't exist
-    user_storage_dir = os.path.join(_CONFIG.STATE_DIR, "rsync-files", asf_uid)
-    await aiofiles.os.makedirs(user_storage_dir, exist_ok=True)
-
-    # Set the target directory to the user's storage directory
-    argv[4] = user_storage_dir
-    _LOGGER.info(f"Modified command: {argv}")
+    # Create the release's storage directory if it doesn't exist
+    await aiofiles.os.makedirs(argv[4], exist_ok=True)
 
     try:
         # Create subprocess to actually run the command
@@ -200,23 +268,27 @@ async def _handle_client(process: asyncssh.SSHServerProcess) -> None:
 
         # Wait for the process to complete
         exit_status = await proc.wait()
+
         # Start a task to process the new files
-        async with db.session() as data:
-            async with data.begin():
-                from atr.tasks.rsync import Analyse
-
-                upload_uuid = str(uuid.uuid4()).replace("-", "")
-                project_name = "test"
-                data.add(
-                    models.Task(
-                        status=models.TaskStatus.QUEUED,
-                        task_type="rsync_analyse",
-                        task_args=Analyse(
-                            asf_uid=asf_uid, upload_uuid=upload_uuid, project_name=project_name
-                        ).model_dump(),
+        # TODO: Turned this off for now, because we need to decide on the workflow
+        start_task = False
+        if start_task:
+            async with db.session() as data:
+                async with data.begin():
+                    from atr.tasks.rsync import Analyse
+
+                    upload_uuid = str(uuid.uuid4()).replace("-", "")
+                    project_name = "test"
+                    data.add(
+                        models.Task(
+                            status=models.TaskStatus.QUEUED,
+                            task_type="rsync_analyse",
+                            task_args=Analyse(
+                                asf_uid=asf_uid, upload_uuid=upload_uuid, project_name=project_name
+                            ).model_dump(),
+                        )
                     )
-                )
-        # Exit the SSH process with the same status
+        # Exit the SSH process with the same status as the rsync process
         process.exit(exit_status)
 
     except Exception as e:
diff --git a/atr/templates/files-add.html b/atr/templates/files-add.html
index f2e77f3..95891d8 100644
--- a/atr/templates/files-add.html
+++ b/atr/templates/files-add.html
@@ -25,6 +25,42 @@
     </ul>
   </p>
 
+  <h2>Editable ongoing releases</h2>
+  <div class="row row-cols-1 row-cols-md-2 g-4 mb-5">
+    {% for release in editable_releases %}
+      <div class="col" id="{{ release.name }}">
+        <div class="card h-100">
+          <div class="card-body position-relative">
+            <div class="position-absolute top-0 end-0 m-2">
+              <span class="badge bg-success">Ongoing</span>
+            </div>
+            <h5 class="card-title">{{ release.project.display_name }} {{ release.version }}</h5>
+            {% if release.project.committee %}
+              <h6 class="card-subtitle mb-2 text-muted">{{ release.project.committee.display_name }}</h6>
+            {% endif %}
+            <p class="card-text">
+              {% if number_of_release_files(release) > 0 %}
+                This editable ongoing release has {{ number_of_release_files(release) }} file(s).
+              {% else %}
+                This editable ongoing release doesn't have any files yet.
+              {% endif %}
+              Use the command below to add or modify files in this release:
+            </p>
+          </div>
+          <pre class="card-footer bg-light border-1 pt-4 small">
+rsync -av -e 'ssh -p 2222' your/files/ \
+    {{ asf_id }}@{{ server_domain }}:/{{ release.project.name }}/{{ release.version }}/
+            </pre>
+        </div>
+      </div>
+    {% endfor %}
+    {% if editable_releases|length == 0 %}
+      <div class="col-12">
+        <div class="alert alert-info">There are currently no editable ongoing releases.</div>
+      </div>
+    {% endif %}
+  </div>
+
   <h2>Projects</h2>
   <div class="row row-cols-1 row-cols-md-2 g-4 mb-5">
     {% for project in projects %}
@@ -44,16 +80,18 @@
               {% endif %}
               <p class="card-text">
                 {% if editable_releases|length > 0 %}
-                  This project already has ongoing releases:
-                  {% for release in editable_releases %}<a href="#{{ release.name }}">{{ release.version }}</a>;{% endfor %}
-                  to create another one, use the command below.
+                  This project already has the ongoing releases
+                  {% for release in editable_releases %}
+                    <code><a href="#{{ release.name }}" class="border rounded px-2">{{ release.version }}</a></code>,
+                  {% endfor %}
+                  but to create another one, use the command below.
                 {% else %}
                   This project does not have an editable ongoing release.
                   To create one and add files, use the command below.
                 {% endif %}
               </p>
             </div>
-            <pre class="card-footer bg-transparent border-top-0 small">
+            <pre class="card-footer bg-light border-1 pt-4 small">
 rsync -av -e 'ssh -p 2222' your/files/ \
     {{ asf_id }}@{{ server_domain }}:/{{ project.name }}/VERSION/
             </pre>
@@ -72,9 +110,11 @@ rsync -av -e 'ssh -p 2222' your/files/ \
               {% endif %}
               <p class="card-text">
                 {% if editable_releases|length > 0 %}
-                  This project already has ongoing releases:
-                  {% for release in editable_releases %}<a href="#{{ release.name }}">{{ release.version }}</a>;{% endfor %}
-                  to create another one, you must be a member of the project's committee.
+                  This project already has the ongoing releases
+                  {% for release in editable_releases %}
+                    <code><a href="#{{ release.name }}" class="border rounded px-2">{{ release.version }}</a></code>,
+                  {% endfor %}
+                  but to create another one, use the command below.
                 {% else %}
                   This project does not have an editable ongoing release.
                   To create one, you must be a member of the project's committee.
@@ -86,44 +126,4 @@ rsync -av -e 'ssh -p 2222' your/files/ \
       {% endif %}
     {% endfor %}
   </div>
-
-  <h2>Editable ongoing releases</h2>
-  <div class="row row-cols-1 row-cols-md-2 g-4 mb-5">
-    {% set ns = namespace(has_releases=false) %}
-    {% for project in projects %}
-      {% for release in project.editable_releases %}
-        {% set ns.has_releases = true %}
-        <div class="col" id="{{ release.name }}">
-          <div class="card h-100">
-            <div class="card-body position-relative">
-              <div class="position-absolute top-0 end-0 m-2">
-                <span class="badge bg-success">Ongoing</span>
-              </div>
-              <h5 class="card-title">{{ project.display_name }} {{ release.version or "?" }}</h5>
-              {% if project.committee %}
-                <h6 class="card-subtitle mb-2 text-muted">{{ project.committee.display_name }}</h6>
-              {% endif %}
-              <p class="card-text">
-                {% if number_of_release_files(release) > 0 %}
-                  This editable ongoing release has {{ number_of_release_files(release) }} file(s).
-                {% else %}
-                  This editable ongoing release doesn't have any files yet.
-                {% endif %}
-                Use the command below to add or modify files in this release:
-              </p>
-            </div>
-            <pre class="card-footer bg-transparent border-top-0 small">
-rsync -av -e 'ssh -p 2222' your/files/ \
-    {{ asf_id }}@{{ server_domain }}:/{{ project.name }}/{{ release.version }}/
-            </pre>
-          </div>
-        </div>
-      {% endfor %}
-    {% endfor %}
-    {% if not ns.has_releases %}
-      <div class="col-12">
-        <div class="alert alert-info">There are currently no editable ongoing releases.</div>
-      </div>
-    {% endif %}
-  </div>
 {% endblock content %}
diff --git a/atr/user.py b/atr/user.py
new file mode 100644
index 0000000..9b0f7a7
--- /dev/null
+++ b/atr/user.py
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""user.py"""
+
+import atr.db as db
+import atr.db.models as models
+
+
+async def editable_releases(uid: str, user_projects: list[models.Project] | None = None) -> list[models.Release]:
+    if user_projects is None:
+        user_projects = await projects(uid)
+    user_editable_releases: list[models.Release] = []
+    for p in user_projects:
+        releases = await p.editable_releases
+        user_editable_releases.extend(releases)
+    return user_editable_releases
+
+
+def is_committer(committee: models.Committee | None, uid: str) -> bool:
+    if committee is None:
+        return False
+    return any((committer_uid == uid) for committer_uid in committee.committers)
+
+
+def is_committee_member(committee: models.Committee | None, uid: str) -> bool:
+    if committee is None:
+        return False
+    return any((member_uid == uid) for member_uid in committee.committee_members)
+
+
+async def projects(uid: str) -> list[models.Project]:
+    user_projects: list[models.Project] = []
+    async with db.session() as data:
+        # Must have releases, because this is used in editable_releases
+        projects = await data.project(_committee=True, _releases=True).all()
+        for p in projects:
+            if p.committee is None:
+                continue
+            if (uid in p.committee.committee_members) or (uid in p.committee.committers):
+                user_projects.append(p)
+    return user_projects
+
+
+async def releases(uid: str) -> list[models.Release]:
+    user_releases: list[models.Release] = []
+    async with db.session() as data:
+        # TODO: We're limiting this to candidates
+        # We should either call this user_candidate_releases, or change the query
+        releases = await data.release(stage=models.ReleaseStage.CANDIDATE, _project=True, _committee=True).all()
+        user_releases = []
+        for r in releases:
+            if r.committee is None:
+                continue
+            if (uid in r.committee.committee_members) or (uid in r.committee.committers):
+                user_releases.append(r)
+    return user_releases


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org