You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ak...@apache.org on 2024/03/26 10:43:01 UTC
(buildstream) branch abderrahim/buildstream-mirrors-merge2 created (now 054ccc755)
This is an automated email from the ASF dual-hosted git repository.
akitouni pushed a change to branch abderrahim/buildstream-mirrors-merge2
in repository https://gitbox.apache.org/repos/asf/buildstream.git
at 054ccc755 Merge branch 'juerg/junction-aliases' into abderrahim/buildstream-mirrors-merge2
This branch includes the following new commits:
new ce48829eb DownloadableFileSource: Support "auth-header-format" extra data
new 3d692cfc5 tests: Added test for bearer authorization in DownloadableFileSource
new 5ca97f7d3 Merge branch 'abderrahim/downloadable-auth' into abderrahim/buildstream-mirrors-merge2
new 054ccc755 Merge branch 'juerg/junction-aliases' into abderrahim/buildstream-mirrors-merge2
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(buildstream) 01/04: DownloadableFileSource: Support "auth-header-format" extra data
Posted by ak...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
akitouni pushed a commit to branch abderrahim/buildstream-mirrors-merge2
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ce48829ebd8865d509a732eac9d10e8d44678e3c
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Tue Feb 27 18:33:35 2024 +0900
DownloadableFileSource: Support "auth-header-format" extra data
Allow SourceMirror plugins to provide information to construct auth
headers for downloadable files.
---
src/buildstream/downloadablefilesource.py | 100 ++++++++++++++++++++++++++++--
1 file changed, 96 insertions(+), 4 deletions(-)
diff --git a/src/buildstream/downloadablefilesource.py b/src/buildstream/downloadablefilesource.py
index f28c230d8..77e2bfa1c 100644
--- a/src/buildstream/downloadablefilesource.py
+++ b/src/buildstream/downloadablefilesource.py
@@ -25,6 +25,66 @@ Any derived classes must write their own stage() and get_unique_key()
implementation.
+SourceMirror extra data "auth-header-format"
+--------------------------------------------
+The DownloadableFileSource, and consequently any :class:`Source <buildstream.source.Source>`
+implementations which derive from DownloadableFileSource, support the "auth-header-format"
+extra data returned by :class:`SourceMirror <buildstream.sourcemirror.SourceMirror>` plugins
+through :func:`Source.translate_url() <buildstream.source.Source.translate_url>`.
+
+This functionality is available **Since: 2.2**.
+
+This allows one to use :class:`SourceMirror <buildstream.sourcemirror.SourceMirror>` plugins
+to add an authorization header to the ``GET`` requests.
+
+
+**Example:**
+
+.. code:: python
+
+ class MySourceMirror(SourceMirror):
+
+ def translate_url(
+ self,
+ *,
+ project_name: str,
+ alias: str,
+ alias_url: str,
+ alias_substitute_url: Optional[str],
+ source_url: str,
+ extra_data: Optional[Dict[str, Any]],
+ ) -> str:
+
+ #
+ # Set the "auth-header-format" extra data
+ #
+ if extra_data is not None:
+ extra_data["auth-header-format"] = "Bearer {password}"
+
+ return alias_substitute_url + source_url
+
+The "auth-header-format" value **must** contain the ``{password}`` expression, which
+will be substituted with the corresponding password found in the user's ``~/.netrc``.
+
+
+**Example:**
+
+If the URL reported by :func:`SourceMirror.translate_url() <buildstream.sourcemirror.SourceMirror.translate_url>`
+is ``http://flying-ponies.com/downloads/pony.tgz``, then a corresponding entry will be expected in the
+user's ``~/.netrc``:
+
+.. code::
+
+ flying-ponies.com
+ password 1234
+
+Assuming the ``"auth-header-format"`` value of ``Bearer {password}`` and the configured password ``1234``,
+the DownloadableFileSource will add the following header to the ``GET`` request to download the file:
+
+.. code::
+
+ Authorization: Bearer 1234
+
"""
@@ -95,6 +155,22 @@ def _download_file(opener_creator, url, etag, directory):
request.add_header("Accept", "*/*")
request.add_header("User-Agent", "BuildStream/2")
+ if opener_creator.auth_header_format:
+ if not opener_creator.netrc_config:
+ raise SourceError("Authorization header format specified without supporting netrc")
+
+ parts = urllib.parse.urlsplit(url)
+ entry = opener_creator.netrc_config.authenticators(parts.hostname)
+ if not entry:
+ raise SourceError(
+ "Authorization header format specified without provided password",
+ detail="No password specified in netrc for hostname: {}".format(parts.hostname),
+ )
+
+ _, _, password = entry
+ auth_header = opener_creator.auth_header_format.format(password=password)
+ request.add_header("Authorization", auth_header)
+
if etag is not None:
request.add_header("If-None-Match", etag)
@@ -145,7 +221,22 @@ class DownloadableFileSource(Source):
def configure(self, node):
self.original_url = node.get_str("url")
self.ref = node.get_str("ref", None)
- self.url = self.translate_url(self.original_url)
+
+ extra_data = {}
+ self.url = self.translate_url(self.original_url, extra_data=extra_data)
+ self.auth_header_format = extra_data.get("auth-header-format")
+
+ #
+ # Validate the auth header format for a `{password}` formatting identifier
+ #
+ if self.auth_header_format:
+ try:
+ self.auth_header_format.format(password="dummy")
+ except KeyError as e:
+ raise SourceError(
+ "SourceMirror specified auth-header-format without a password", detail=self.auth_header_format
+ ) from e
+
self._mirror_dir = os.path.join(self.get_mirror_directory(), utils.url_directory_name(self.original_url))
def preflight(self):
@@ -227,7 +318,7 @@ class DownloadableFileSource(Source):
else:
etag = None
- url_opener_creator = _UrlOpenerCreator(self._parse_netrc())
+ url_opener_creator = _UrlOpenerCreator(self._parse_netrc(), self.auth_header_format)
local_file, new_etag, error = self.blocking_activity(
_download_file, (url_opener_creator, self.url, etag, td), activity_name
@@ -283,11 +374,12 @@ class DownloadableFileSource(Source):
class _UrlOpenerCreator:
- def __init__(self, netrc_config):
+ def __init__(self, netrc_config, auth_header_format):
self.netrc_config = netrc_config
+ self.auth_header_format = auth_header_format
def get_url_opener(self):
- if self.netrc_config:
+ if not self.auth_header_format and self.netrc_config:
netrc_pw_mgr = _NetrcPasswordManager(self.netrc_config)
http_auth = urllib.request.HTTPBasicAuthHandler(netrc_pw_mgr)
ftp_handler = _NetrcFTPOpener(self.netrc_config)
(buildstream) 04/04: Merge branch 'juerg/junction-aliases' into abderrahim/buildstream-mirrors-merge2
Posted by ak...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
akitouni pushed a commit to branch abderrahim/buildstream-mirrors-merge2
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 054ccc7552f95588adc371d7228963840fa083f1
Merge: 5ca97f7d3 1cb4c9fb9
Author: Abderrahim Kitouni <ab...@codethink.co.uk>
AuthorDate: Mon Mar 25 18:10:49 2024 +0100
Merge branch 'juerg/junction-aliases' into abderrahim/buildstream-mirrors-merge2
src/buildstream/_project.py | 38 +++++++++++++++++++++++++++-
src/buildstream/plugins/elements/junction.py | 8 +++++-
2 files changed, 44 insertions(+), 2 deletions(-)
diff --cc src/buildstream/_project.py
index 1fa8ea545,6ae488731..da0e914ee
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@@ -238,8 -215,23 +239,19 @@@ class Project
if url and utils._ALIAS_SEPARATOR in url:
url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
- alias_url = config._aliases.get_str(url_alias, default=None)
+ alias_url = self.get_alias_url(url_alias, first_pass=first_pass)
if alias_url:
+ if self.junction:
+ parent_project = self.junction._get_project()
+ parent_alias = self.junction.aliases.get_str(url_alias, default=None)
+ if parent_alias:
+ # Delegate translation to parent project
+ return parent_project.translate_url(
+ parent_alias + utils._ALIAS_SEPARATOR + url_body, first_pass=first_pass
+ )
+ elif not parent_project.allow_subproject_uris:
+ return url
+
url = alias_url + url_body
return url
@@@ -422,7 -416,16 +442,16 @@@
if not alias or alias not in config._aliases: # pylint: disable=unsupported-membership-test
return [None]
+ if self.junction:
+ parent_project = self.junction._get_project()
+ parent_alias = self.junction.aliases.get_str(alias, default=None)
+ if parent_alias:
+ # Delegate translation to parent project
+ return parent_project.get_alias_uris(parent_alias, first_pass=first_pass, tracking=tracking)
+ elif not parent_project.allow_subproject_uris:
+ return [None]
+
- uri_list = []
+ uri_list: List[Optional[SourceMirror | str]] = []
policy = self._context.track_source if tracking else self._context.fetch_source
if policy in (_SourceUriPolicy.ALL, _SourceUriPolicy.MIRRORS) or (
(buildstream) 02/04: tests: Added test for bearer authorization in DownloadableFileSource
Posted by ak...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
akitouni pushed a commit to branch abderrahim/buildstream-mirrors-merge2
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3d692cfc560ff1aa699c4ff060380aa9ba427d1a
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Tue Feb 27 18:34:42 2024 +0900
tests: Added test for bearer authorization in DownloadableFileSource
---
tests/sources/tar.py | 70 ++++++++++++-
.../tar/fetch/sourcemirrors/bearermirror.py | 29 ++++++
tests/testutils/bearer_http_server.py | 116 +++++++++++++++++++++
tests/testutils/file_server.py | 17 +++
4 files changed, 231 insertions(+), 1 deletion(-)
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index c7abbc6dd..8712c77eb 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -29,7 +29,7 @@ from buildstream.exceptions import ErrorDomain
from buildstream._testing import generate_project, generate_element
from buildstream._testing import cli # pylint: disable=unused-import
from buildstream._testing._utils.site import HAVE_LZIP
-from tests.testutils.file_server import create_file_server
+from tests.testutils.file_server import create_file_server, create_bearer_http_server
from . import list_dir_contents
DATA_DIR = os.path.join(
@@ -355,6 +355,74 @@ def test_use_netrc(cli, datafiles, server_type, tmpdir):
assert checkout_contents == original_contents
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
+def test_use_netrc_bearer_auth(cli, datafiles, tmpdir):
+ file_server_files = os.path.join(str(tmpdir), "file_server")
+ fake_home = os.path.join(str(tmpdir), "fake_home")
+ os.makedirs(file_server_files, exist_ok=True)
+ os.makedirs(fake_home, exist_ok=True)
+ project = str(datafiles)
+ checkoutdir = os.path.join(str(tmpdir), "checkout")
+
+ os.environ["HOME"] = fake_home
+ with open(os.path.join(fake_home, ".netrc"), "wb") as f:
+ os.fchmod(f.fileno(), 0o700)
+ f.write(b"machine 127.0.0.1\n")
+ f.write(b"password 12345\n")
+
+ #
+ # Enable using mirrors for source tracking
+ #
+ cli.configure({"track": {"source": "mirrors"}})
+
+ #
+ # Create a file server which uses bearer authentication
+ #
+ with create_bearer_http_server() as server:
+ server.set_directory(file_server_files)
+ server.add_token("12345")
+
+ #
+ # Configure the project to load our source mirror plugin which
+ # reports the "auth-header-format" extra data
+ #
+ additional_config = {
+ "aliases": {"tmpdir": server.base_url()},
+ "mirrors": [
+ {
+ "name": "middle-earth",
+ "kind": "bearermirror",
+ "aliases": {
+ "tmpdir": [server.base_url()],
+ },
+ },
+ ],
+ "plugins": [
+ {"origin": "local", "path": "sourcemirrors", "source-mirrors": ["bearermirror"]},
+ ],
+ }
+ generate_project(project, config=additional_config)
+
+ src_tar = os.path.join(file_server_files, "a.tar.gz")
+ _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
+
+ server.start()
+
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
+ result.assert_success()
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
+ result.assert_success()
+ result = cli.run(project=project, args=["build", "target.bst"])
+ result.assert_success()
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
+ result.assert_success()
+
+ original_dir = os.path.join(str(datafiles), "content", "a")
+ original_contents = list_dir_contents(original_dir)
+ checkout_contents = list_dir_contents(checkoutdir)
+ assert checkout_contents == original_contents
+
+
@pytest.mark.parametrize("server_type", ("FTP", "HTTP"))
@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir):
diff --git a/tests/sources/tar/fetch/sourcemirrors/bearermirror.py b/tests/sources/tar/fetch/sourcemirrors/bearermirror.py
new file mode 100644
index 000000000..54d930561
--- /dev/null
+++ b/tests/sources/tar/fetch/sourcemirrors/bearermirror.py
@@ -0,0 +1,29 @@
+from typing import Optional, Dict, Any
+
+from buildstream import SourceMirror, MappingNode
+
+
+class Sample(SourceMirror):
+ BST_MIN_VERSION = "2.0"
+
+ def translate_url(
+ self,
+ *,
+ project_name: str,
+ alias: str,
+ alias_url: str,
+ alias_substitute_url: Optional[str],
+ source_url: str,
+ extra_data: Optional[Dict[str, Any]],
+ ) -> str:
+
+ if extra_data is not None:
+ extra_data["auth-header-format"] = "Bearer {password}"
+
+ return alias_substitute_url + source_url
+
+
+# Plugin entry point
+def setup():
+
+ return Sample
diff --git a/tests/testutils/bearer_http_server.py b/tests/testutils/bearer_http_server.py
new file mode 100644
index 000000000..676098385
--- /dev/null
+++ b/tests/testutils/bearer_http_server.py
@@ -0,0 +1,116 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import multiprocessing
+import os
+import posixpath
+import html
+from http.server import SimpleHTTPRequestHandler, HTTPServer, HTTPStatus
+
+
+class Unauthorized(Exception):
+ pass
+
+
+class BearerRequestHandler(SimpleHTTPRequestHandler):
+ def get_root_dir(self):
+ authorization = self.headers.get("authorization")
+ if not authorization:
+ raise Unauthorized("unauthorized")
+
+ authorization = authorization.split()
+ if len(authorization) != 2 or authorization[0].lower() != "bearer":
+ raise Unauthorized("unauthorized")
+
+ token = authorization[1]
+ if token not in self.server.tokens:
+ raise Unauthorized("unauthorized")
+
+ return self.server.directory
+
+ def unauthorized(self):
+ shortmsg, longmsg = self.responses[HTTPStatus.UNAUTHORIZED]
+ self.send_response(HTTPStatus.UNAUTHORIZED, shortmsg)
+ self.send_header("Connection", "close")
+
+ content = self.error_message_format % {
+ "code": HTTPStatus.UNAUTHORIZED,
+ "message": html.escape(longmsg, quote=False),
+ "explain": html.escape(longmsg, quote=False),
+ }
+ body = content.encode("UTF-8", "replace")
+ self.send_header("Content-Type", self.error_content_type)
+ self.send_header("Content-Length", str(len(body)))
+ self.send_header("WWW-Authenticate", 'Bearer realm="{}"'.format(self.server.realm))
+ self.end_headers()
+ self.end_headers()
+
+ if self.command != "HEAD" and body:
+ self.wfile.write(body)
+
+ def do_GET(self):
+ try:
+ super().do_GET()
+ except Unauthorized:
+ self.unauthorized()
+
+ def do_HEAD(self):
+ try:
+ super().do_HEAD()
+ except Unauthorized:
+ self.unauthorized()
+
+ def translate_path(self, path):
+ path = path.split("?", 1)[0]
+ path = path.split("#", 1)[0]
+ path = posixpath.normpath(path)
+ assert posixpath.isabs(path)
+ path = posixpath.relpath(path, "/")
+ return os.path.join(self.get_root_dir(), path)
+
+
+class BearerHTTPServer(HTTPServer):
+ def __init__(self, *args, **kwargs):
+ self.tokens = set()
+ self.directory = None
+ self.realm = "Realm"
+ super().__init__(*args, **kwargs)
+
+
+class BearerHttpServer(multiprocessing.Process):
+ def __init__(self):
+ super().__init__()
+ self.server = BearerHTTPServer(("127.0.0.1", 0), BearerRequestHandler)
+ self.started = False
+
+ def start(self):
+ self.started = True
+ super().start()
+
+ def run(self):
+ self.server.serve_forever()
+
+ def stop(self):
+ if not self.started:
+ return
+ self.terminate()
+ self.join()
+
+ def set_directory(self, directory):
+ self.server.directory = directory
+
+ def add_token(self, token):
+ self.server.tokens.add(token)
+
+ def base_url(self):
+ return "http://127.0.0.1:{}".format(self.server.server_port)
diff --git a/tests/testutils/file_server.py b/tests/testutils/file_server.py
index ac1d6ecf6..1d43edbe9 100644
--- a/tests/testutils/file_server.py
+++ b/tests/testutils/file_server.py
@@ -15,6 +15,7 @@ from contextlib import contextmanager
from .ftp_server import SimpleFtpServer
from .http_server import SimpleHttpServer
+from .bearer_http_server import BearerHttpServer
@contextmanager
@@ -30,3 +31,19 @@ def create_file_server(file_server_type):
yield server
finally:
server.stop()
+
+
+#
+# We use a separate function here in order to avoid
+# confusing the linter (which thinks that anything
+# yielded by `create_file_server()` is a SimpleFtpServer).
+#
+# And no, type annotations with Union[...] does not fix this.
+#
+@contextmanager
+def create_bearer_http_server():
+ server = BearerHttpServer()
+ try:
+ yield server
+ finally:
+ server.stop()
(buildstream) 03/04: Merge branch 'abderrahim/downloadable-auth' into abderrahim/buildstream-mirrors-merge2
Posted by ak...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
akitouni pushed a commit to branch abderrahim/buildstream-mirrors-merge2
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5ca97f7d3702f46fd08ca34713bd8b0c20c8cfbe
Merge: d3c92095e 3d692cfc5
Author: Abderrahim Kitouni <ab...@codethink.co.uk>
AuthorDate: Mon Mar 25 18:09:47 2024 +0100
Merge branch 'abderrahim/downloadable-auth' into abderrahim/buildstream-mirrors-merge2
src/buildstream/downloadablefilesource.py | 100 +++++++++++++++++-
tests/sources/tar.py | 70 ++++++++++++-
.../tar/fetch/sourcemirrors/bearermirror.py | 29 ++++++
tests/testutils/bearer_http_server.py | 116 +++++++++++++++++++++
tests/testutils/file_server.py | 17 +++
5 files changed, 327 insertions(+), 5 deletions(-)