You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2024/10/14 09:37:05 UTC
(buildstream-plugins) 04/04: Adding gitlab_lfs_mirror source mirror plugin
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/mirror-plugins
in repository https://gitbox.apache.org/repos/asf/buildstream-plugins.git
commit a312599ee45ed0b11a896eefc392848d091bb75c
Author: Tristan van Berkom <tr...@upstairslabs.com>
AuthorDate: Mon Oct 14 18:28:56 2024 +0900
Adding gitlab_lfs_mirror source mirror plugin
Since we've added the SourceMirror feature in BuildStream 2.2, it makes
sense to add this mirror plugin upstream.
This is especially helpful as we can rely on loading stable
buildstream-plugins via the `pip` plugin origin for loading source
mirror plugins, avoiding a cyclic dependency issue if you want to
use mirror plugins to load plugins via the `junction` plugin origin.
---
doc/source/index.rst | 1 +
project.conf | 1 +
setup.py | 1 +
.../sourcemirrors/gitlab_lfs_mirror.py | 76 ++++++++++++++++++++++
4 files changed, 79 insertions(+)
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 95b6f8c..88c5bc5 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -48,3 +48,4 @@ To these plugins in your project, follow the
:caption: Source Mirror Plugins
sourcemirrors/simple_mirror
+ sourcemirrors/gitlab_lfs_mirror
diff --git a/project.conf b/project.conf
index 301bb08..bad78fe 100644
--- a/project.conf
+++ b/project.conf
@@ -43,4 +43,5 @@ plugins:
- origin: local
path: src/buildstream_plugins/sourcemirrors
source-mirrors:
+ - gitlab_lfs_mirror
- simple_mirror
diff --git a/setup.py b/setup.py
index b7de25b..5417244 100755
--- a/setup.py
+++ b/setup.py
@@ -93,6 +93,7 @@ setup(
"zip = buildstream_plugins.sources.zip",
],
"buildstream.plugins.sourcemirrors": [
+ "gitlab_lfs_mirror = buildstream_plugins.sourcemirrors.gitlab_lfs_mirror",
"simple_mirror = buildstream_plugins.sourcemirrors.simple_mirror",
],
},
diff --git a/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py b/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py
new file mode 100644
index 0000000..0d30632
--- /dev/null
+++ b/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py
@@ -0,0 +1,76 @@
+"""
+gitlab_lfs_mirror - plugin for accessing files stored in git-lfs on GitLab
+==========================================================================
+
+.. note::
+
+ The ``gitlab_lfs_mirror`` plugin is available *Since 2.4.0*
+
+If you store files in a git-lfs repository on gitlab.com, you can access
+them using a URL like
+
+https://gitlab.com/path/to/repo/-/raw/master/path/to/file
+
+which you can then use as a mirror for buildstream. However this only works for
+public repositories. For internal (on self-hosted GitLab instances) and private
+repositories, the above doesn't work since buildstream cannot authenticate with
+the GitLab web UI.
+
+This plugin solves this by going through the GitLab REST API. You need an
+access token that access the Repository Files API (i.e. with read_api or
+read_repository scope). As of this writing, the GitLab CI/CD job token doesn't
+allow access to this API endpoint, so you need a dedicated access token.
+
+**Usage:**
+
+.. code:: yaml
+
+- name: my-mirror
+ kind: gitlab_lfs_mirror
+ config:
+ url: https://gitlab.example.com/
+ project: mirrors/{alias}
+ ref: main # optional, defaults to master
+ aliases:
+ - my-alias
+ - another-alias
+"""
+
+from posixpath import join
+from urllib.parse import quote
+
+from buildstream import SourceMirror
+
+
+class GitlabLFSMirror(SourceMirror):
+ BST_MIN_VERSION = "2.2"
+
+ def configure(self, node):
+ node.validate_keys(["aliases", "url", "project", "ref"])
+ self.set_supported_aliases(node.get_str_list("aliases"))
+
+ self.url = node.get_str("url")
+ self.project = node.get_str("project")
+ self.ref = node.get_str("ref", "master")
+
+ def translate_url(self, alias, alias_url, source_url, extra_data):
+ project_id = quote(self.project.format(alias=alias), safe="")
+ filename = quote(source_url, safe="")
+
+ translated_url = join(
+ self.url,
+ "api/v4/projects",
+ project_id,
+ "repository/files",
+ filename,
+ f"raw?ref={self.ref}&lfs=true",
+ )
+
+ if extra_data is not None:
+ extra_data["http-auth"] = "bearer"
+
+ return translated_url
+
+
+def setup():
+ return GitlabLFSMirror