You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2022/02/17 04:29:35 UTC
[buildstream] branch bst-1 updated: Backport ostree changes from bst-plugins-experimental
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch bst-1
in repository https://gitbox.apache.org/repos/asf/buildstream.git
The following commit(s) were added to refs/heads/bst-1 by this push:
new a0551c4 Backport ostree changes from bst-plugins-experimental
new 8699d3f Merge pull request #1587 from nanonyme/nanonyme/ostree-source
a0551c4 is described below
commit a0551c45eefe33a94c1fd52d6908e7ce69080c24
Author: Seppo Yli-Olli <se...@gmail.com>
AuthorDate: Wed Feb 9 23:00:56 2022 +0200
Backport ostree changes from bst-plugins-experimental
This drops dependency on PyGObject
---
buildstream/_ostree.py | 289 ----------------------------------
buildstream/plugins/sources/ostree.py | 169 ++++++++++++++------
doc/source/install_linux_distro.rst | 1 -
requirements/plugin-requirements.in | 1 -
requirements/plugin-requirements.txt | 3 -
5 files changed, 123 insertions(+), 340 deletions(-)
diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py
deleted file mode 100644
index 40fe2c2..0000000
--- a/buildstream/_ostree.py
+++ /dev/null
@@ -1,289 +0,0 @@
-#
-# Copyright (C) 2017 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library. If not, see <http://www.gnu.org/licenses/>.
-#
-# Authors:
-# Jürg Billeter <ju...@codethink.co.uk>
-# Andrew Leeming <an...@codethink.co.uk>
-# Tristan Van Berkom <tr...@codethink.co.uk>
-#
-# Code based on Jürg's artifact cache and Andrew's ostree plugin
-#
-
-# Disable pylint warnings that are not appicable to this module
-# pylint: disable=bad-exception-context,catching-non-exception
-
-import os
-
-import gi # pylint: disable=import-error
-from gi.repository.GLib import Variant, VariantDict
-
-from ._exceptions import BstError, ErrorDomain
-
-# pylint: disable=wrong-import-position,wrong-import-order
-gi.require_version('OSTree', '1.0')
-from gi.repository import GLib, Gio, OSTree # nopep8
-
-
-# For users of this file, they must expect (except) it.
-class OSTreeError(BstError):
- def __init__(self, message, reason=None):
- super().__init__(message, domain=ErrorDomain.UTIL, reason=reason)
-
-
-# ensure()
-#
-# Args:
-# path (str): The file path to where the desired repo should be
-# compress (bool): use compression or not when creating
-#
-# Returns: an OSTree.Repo
-def ensure(path, compress):
-
- # create also succeeds on existing repository
- repo = OSTree.Repo.new(Gio.File.new_for_path(path))
- mode = OSTree.RepoMode.ARCHIVE_Z2 if compress \
- else OSTree.RepoMode.BARE_USER
-
- repo.create(mode)
-
- # Disble OSTree's built in minimum-disk-space check.
- config = repo.copy_config()
- config.set_string('core', 'min-free-space-percent', '0')
- repo.write_config(config)
- repo.reload_config()
-
- return repo
-
-
-# checkout()
-#
-# Checkout the content at 'commit' from 'repo' in
-# the specified 'path'
-#
-# Args:
-# repo (OSTree.Repo): The repo
-# path (str): The checkout path
-# commit_ (str): The commit checksum to checkout
-# user (boot): Whether to checkout in user mode
-#
-def checkout(repo, path, commit_, user=False):
-
- # Check out a full copy of an OSTree at a given ref to some directory.
- #
- # Note: OSTree does not like updating directories inline/sync, therefore
- # make sure you checkout to a clean directory or add additional code to support
- # union mode or (if it exists) file replacement/update.
- #
- # Returns True on success
- #
- # cli exmaple:
- # ostree --repo=repo checkout --user-mode runtime/org.freedesktop.Sdk/x86_64/1.4 foo
- os.makedirs(os.path.dirname(path), exist_ok=True)
-
- options = OSTree.RepoCheckoutAtOptions()
-
- # For repos which contain root owned files, we need
- # to checkout with OSTree.RepoCheckoutMode.USER
- #
- # This will reassign uid/gid and also munge the
- # permission bits a bit.
- if user:
- options.mode = OSTree.RepoCheckoutMode.USER
-
- # Using AT_FDCWD value from fcntl.h
- #
- # This will be ignored if the passed path is an absolute path,
- # if path is a relative path then it will be appended to the
- # current working directory.
- AT_FDCWD = -100
- try:
- repo.checkout_at(options, AT_FDCWD, path, commit_)
- except GLib.GError as e:
- raise OSTreeError("Failed to checkout commit '{}': {}".format(commit_, e.message)) from e
-
-
-# exists():
-#
-# Checks wether a given commit or symbolic ref exists and
-# is locally cached in the specified repo.
-#
-# Args:
-# repo (OSTree.Repo): The repo
-# ref (str): A commit checksum or symbolic ref
-#
-# Returns:
-# (bool): Whether 'ref' is valid in 'repo'
-#
-def exists(repo, ref):
-
- # Get the commit checksum, this will:
- #
- # o Return a commit checksum if ref is a symbolic branch
- # o Return the same commit checksum if ref is a valid commit checksum
- # o Return None if the ostree repo doesnt know this ref.
- #
- ref = checksum(repo, ref)
- if ref is None:
- return False
-
- # If we do have a ref which the ostree knows about, this does
- # not mean we necessarily have the object locally (we may just
- # have some metadata about it, this can happen).
- #
- # Use has_object() only with a resolved valid commit checksum
- # to check if we actually have the object locally.
- _, has_object = repo.has_object(OSTree.ObjectType.COMMIT, ref, None)
- return has_object
-
-
-# checksum():
-#
-# Returns the commit checksum for a given symbolic ref,
-# which might be a branch or tag. If it is a branch,
-# the latest commit checksum for the given branch is returned.
-#
-# Args:
-# repo (OSTree.Repo): The repo
-# ref (str): The symbolic ref
-#
-# Returns:
-# (str): The commit checksum, or None if ref does not exist.
-#
-def checksum(repo, ref):
-
- _, checksum_ = repo.resolve_rev(ref, True)
- return checksum_
-
-
-# fetch()
-#
-# Fetch new objects from a remote, if configured
-#
-# Args:
-# repo (OSTree.Repo): The repo
-# remote (str): An optional remote name, defaults to 'origin'
-# ref (str): An optional ref to fetch, will reduce the amount of objects fetched
-# progress (callable): An optional progress callback
-#
-# Note that a commit checksum or a branch reference are both
-# valid options for the 'ref' parameter. Using the ref parameter
-# can save a lot of bandwidth but mirroring the full repo is
-# still possible.
-#
-def fetch(repo, remote="origin", ref=None, progress=None):
- # Fetch metadata of the repo from a remote
- #
- # cli example:
- # ostree --repo=repo pull --mirror freedesktop:runtime/org.freedesktop.Sdk/x86_64/1.4
- def progress_callback(info):
- status = async_progress.get_status()
- outstanding_fetches = async_progress.get_uint('outstanding-fetches')
- bytes_transferred = async_progress.get_uint64('bytes-transferred')
- fetched = async_progress.get_uint('fetched')
- requested = async_progress.get_uint('requested')
-
- if status:
- progress(0.0, status)
- elif outstanding_fetches > 0:
- formatted_bytes = GLib.format_size_full(bytes_transferred, 0)
- if requested == 0:
- percent = 0.0
- else:
- percent = (fetched * 1.0 / requested) * 100
-
- progress(percent,
- "Receiving objects: {:d}% ({:d}/{:d}) {}".format(int(percent), fetched,
- requested, formatted_bytes))
- else:
- progress(100.0, "Writing Objects")
-
- async_progress = None
- if progress is not None:
- async_progress = OSTree.AsyncProgress.new()
- async_progress.connect('changed', progress_callback)
-
- # FIXME: This hangs the process and ignores keyboard interrupt,
- # fix this using the Gio.Cancellable
- refs = None
- if ref is not None:
- refs = [ref]
-
- try:
- repo.pull(remote,
- refs,
- OSTree.RepoPullFlags.MIRROR,
- async_progress,
- None) # Gio.Cancellable
- except GLib.GError as e:
- if ref is not None:
- raise OSTreeError("Failed to fetch ref '{}' from '{}': {}".format(ref, remote, e.message)) from e
- raise OSTreeError("Failed to fetch from '{}': {}".format(remote, e.message)) from e
-
-
-# configure_remote():
-#
-# Ensures a remote is setup to a given url.
-#
-# Args:
-# repo (OSTree.Repo): The repo
-# remote (str): The name of the remote
-# url (str): The url of the remote ostree repo
-# key_url (str): The optional url of a GPG key (should be a local file)
-#
-def configure_remote(repo, remote, url, key_url=None):
- # Add a remote OSTree repo. If no key is given, we disable gpg checking.
- #
- # cli exmaple:
- # wget https://sdk.gnome.org/keys/gnome-sdk.gpg
- # ostree --repo=repo --gpg-import=gnome-sdk.gpg remote add freedesktop https://sdk.gnome.org/repo
- options = None # or GLib.Variant of type a{sv}
- if key_url is None:
- vd = VariantDict.new()
- vd.insert_value('gpg-verify', Variant.new_boolean(False))
- options = vd.end()
-
- try:
- repo.remote_change(None, # Optional OSTree.Sysroot
- OSTree.RepoRemoteChange.ADD_IF_NOT_EXISTS,
- remote, # Remote name
- url, # Remote url
- options, # Remote options
- None) # Optional Gio.Cancellable
- except GLib.GError as e:
- raise OSTreeError("Failed to configure remote '{}': {}".format(remote, e.message)) from e
-
- # Remote needs to exist before adding key
- if key_url is not None:
- try:
- gfile = Gio.File.new_for_uri(key_url)
- stream = gfile.read()
-
- # In ostree commit `v2019.2-10-gaa5df899`, the python
- # facing API was changed by way of modifying the
- # instrospection annotations.
- #
- # This means we need to call this API in two different
- # ways depending on which ostree version is installed.
- #
- try:
- # New API
- repo.remote_gpg_import(remote, stream, None, None)
- except TypeError:
- # Old API
- repo.remote_gpg_import(remote, stream, None, 0, None)
-
- except GLib.GError as e:
- raise OSTreeError("Failed to add gpg key from url '{}': {}".format(key_url, e.message)) from e
diff --git a/buildstream/plugins/sources/ostree.py b/buildstream/plugins/sources/ostree.py
index 526a91a..abcb16a 100644
--- a/buildstream/plugins/sources/ostree.py
+++ b/buildstream/plugins/sources/ostree.py
@@ -54,9 +54,7 @@ import os
import shutil
from buildstream import Source, SourceError, Consistency
-from buildstream import _ostree
from buildstream import utils
-from buildstream._ostree import OSTreeError
class OSTreeSource(Source):
@@ -66,6 +64,8 @@ class OSTreeSource(Source):
self.node_validate(node, ['url', 'ref', 'track', 'gpg-key'] + Source.COMMON_CONFIG_KEYS)
+ self.ostree = None
+
self.original_url = self.node_get_member(node, str, 'url')
self.url = self.translate_url(self.original_url)
self.ref = self.node_get_member(node, str, 'ref', None)
@@ -90,7 +90,8 @@ class OSTreeSource(Source):
self.repo = None
def preflight(self):
- pass
+ # Check if ostree is installed, get the binary at the same time
+ self.ostree = utils.get_host_tool("ostree")
def get_unique_key(self):
return [self.original_url, self.ref]
@@ -105,53 +106,95 @@ class OSTreeSource(Source):
node['ref'] = self.ref = ref
def track(self):
- # If self.tracking is not specified its' not an error, just silently return
+ # If self.tracking is not specified it's not an error, just silently return
if not self.tracking:
return None
self.ensure()
remote_name = self.ensure_remote(self.url)
- with self.timed_activity("Fetching tracking ref '{}' from origin: {}"
- .format(self.tracking, self.url)):
- try:
- _ostree.fetch(self.repo, remote=remote_name, ref=self.tracking, progress=self.progress)
- except OSTreeError as e:
- raise SourceError("{}: Failed to fetch tracking ref '{}' from origin {}\n\n{}"
- .format(self, self.tracking, self.url, e)) from e
+ with self.timed_activity(
+ "Fetching tracking ref '{}' from origin: {}".format(
+ self.tracking, self.url
+ )
+ ):
+ self.call(
+ [
+ self.ostree,
+ "pull",
+ "--repo",
+ self.mirror,
+ "--mirror",
+ remote_name,
+ self.tracking,
+ ],
+ fail="Failed to fetch tracking ref '{}' from origin {}".format(
+ self.tracking, self.url
+ ),
+ )
+
+ return self.check_output(
+ [self.ostree, "rev-parse", "--repo", self.mirror, self.tracking],
+ fail="Failed to compute checksum of '{}' on '{}'".format(
+ self.tracking, self.mirror
+ ),
+ )[1]
- return _ostree.checksum(self.repo, self.tracking)
def fetch(self):
self.ensure()
+
remote_name = self.ensure_remote(self.url)
- if not _ostree.exists(self.repo, self.ref):
- with self.timed_activity("Fetching remote ref: {} from origin: {}"
- .format(self.ref, self.url)):
- try:
- _ostree.fetch(self.repo, remote=remote_name, ref=self.ref, progress=self.progress)
- except OSTreeError as e:
- raise SourceError("{}: Failed to fetch ref '{}' from origin: {}\n\n{}"
- .format(self, self.ref, self.url, e)) from e
+ with self.timed_activity(
+ "Fetching remote ref: {} from origin: {}".format(
+ self.ref, self.url
+ )
+ ):
+ self.call(
+ [
+ self.ostree,
+ "pull",
+ "--repo",
+ self.mirror,
+ "--mirror",
+ remote_name,
+ self.ref,
+ ],
+ fail="Failed to fetch ref '{}' from origin: {}".format(
+ self.ref, remote_name
+ ),
+ )
+
def stage(self, directory):
+
self.ensure()
# Checkout self.ref into the specified directory
with self.tempdir() as tmpdir:
- checkoutdir = os.path.join(tmpdir, 'checkout')
-
- with self.timed_activity("Staging ref: {} from origin: {}"
- .format(self.ref, self.url)):
- try:
- _ostree.checkout(self.repo, checkoutdir, self.ref, user=True)
- except OSTreeError as e:
- raise SourceError("{}: Failed to checkout ref '{}' from origin: {}\n\n{}"
- .format(self, self.ref, self.url, e)) from e
+ checkoutdir = os.path.join(tmpdir, "checkout")
+
+ with self.timed_activity(
+ "Staging ref: {} from origin: {}".format(self.ref, self.url)
+ ):
+ self.call(
+ [
+ self.ostree,
+ "checkout",
+ "--repo",
+ self.mirror,
+ "--user-mode",
+ self.ref,
+ checkoutdir,
+ ],
+ fail="Failed to checkout ref '{}' from origin: {}".format(
+ self.ref, self.url
+ ),
+ )
# The target directory is guaranteed to exist, here we must move the
# content of out checkout into the existing target directory.
#
- # We may not be able to create the target directory as it's parent
+ # We may not be able to create the target directory as its parent
# may be readonly, and the directory itself is often a mount point.
#
try:
@@ -159,45 +202,79 @@ class OSTreeSource(Source):
source_path = os.path.join(checkoutdir, entry)
shutil.move(source_path, directory)
except (shutil.Error, OSError) as e:
- raise SourceError("{}: Failed to move ostree checkout {} from '{}' to '{}'\n\n{}"
- .format(self, self.url, tmpdir, directory, e)) from e
+ raise SourceError(
+ "{}: Failed to move ostree checkout {} from '{}' to '{}'\n\n{}".format(
+ self, self.url, tmpdir, directory, e
+ )
+ ) from e
+
def get_consistency(self):
if self.ref is None:
return Consistency.INCONSISTENT
+ elif os.path.exists(self.mirror):
+ if self.call([self.ostree, "show", "--repo", self.mirror, self.ref]) == 0:
+ return Consistency.CACHED
- self.ensure()
- if _ostree.exists(self.repo, self.ref):
- return Consistency.CACHED
return Consistency.RESOLVED
#
# Local helpers
#
def ensure(self):
- if not self.repo:
+ if not os.path.exists(self.mirror):
self.status("Creating local mirror for {}".format(self.url))
+ self.call(
+ [
+ self.ostree,
+ "init",
+ "--repo",
+ self.mirror,
+ "--mode",
+ "archive-z2",
+ ],
+ fail="Unable to create local mirror for repository",
+ )
+ self.call(
+ [
+ self.ostree,
+ "config",
+ "--repo",
+ self.mirror,
+ "set",
+ "core.min-free-space-percent",
+ "0",
+ ],
+ fail="Unable to disable minimum disk space checks",
+ )
- self.repo = _ostree.ensure(self.mirror, True)
def ensure_remote(self, url):
if self.original_url == self.url:
- remote_name = 'origin'
+ remote_name = "origin"
else:
remote_name = utils.url_directory_name(url)
- gpg_key = None
+ command = [
+ self.ostree,
+ "remote",
+ "add",
+ "--if-not-exists",
+ "--repo",
+ self.mirror,
+ remote_name,
+ url,
+ ]
+
if self.gpg_key_path:
- gpg_key = 'file://' + self.gpg_key_path
+ command.extend(["--gpg-import", self.gpg_key_path])
+ else:
+ command.extend(["--no-gpg-verify"])
+
+ self.call(command, fail="Failed to configure origin {}".format(url))
- try:
- _ostree.configure_remote(self.repo, remote_name, url, key_url=gpg_key)
- except OSTreeError as e:
- raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e
return remote_name
- def progress(self, percent, message):
- self.status(message)
# Plugin entry point
diff --git a/doc/source/install_linux_distro.rst b/doc/source/install_linux_distro.rst
index b30ab5a..f27aab5 100644
--- a/doc/source/install_linux_distro.rst
+++ b/doc/source/install_linux_distro.rst
@@ -44,7 +44,6 @@ BuildStream requires the following base system requirements:
* libostree >= v2017.8 with introspection data
* bubblewrap >= 0.1.2
* fuse2
-* PyGObject introspection bindings
* psutil python library (so you don't have to install GCC and python-devel to build it yourself)
BuildStream also depends on the host tools for the :mod:`Source <buildstream.source>` plugins.
diff --git a/requirements/plugin-requirements.in b/requirements/plugin-requirements.in
index e352728..a30105a 100644
--- a/requirements/plugin-requirements.in
+++ b/requirements/plugin-requirements.in
@@ -1,2 +1 @@
arpy
-PyGObject
diff --git a/requirements/plugin-requirements.txt b/requirements/plugin-requirements.txt
index 301094f..ec44c8c 100644
--- a/requirements/plugin-requirements.txt
+++ b/requirements/plugin-requirements.txt
@@ -1,4 +1 @@
arpy==2.2.0
-PyGObject==3.42.0
-## The following requirements were added by pip freeze:
-pycairo==1.20.1