You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 08:13:57 UTC
[buildstream] 01/04: Add SourceTransform base class
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch gokcen/source_transform
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 866c4ca2291db1b66ab6d48f42ae2bff619c917b
Author: Gökçen Nurlu <gn...@bloomberg.net>
AuthorDate: Thu Jun 7 18:28:58 2018 +0100
Add SourceTransform base class
This adds a new base plugin, an alternative to `Source`, that is more suitable
to implement plugins for languages with package-managers.
---
buildstream/__init__.py | 1 +
buildstream/_scheduler/fetchqueue.py | 6 +-
buildstream/element.py | 11 ++-
buildstream/source.py | 4 +-
buildstream/sourcetransform.py | 136 +++++++++++++++++++++++++++++++++++
doc/source/core_framework.rst | 1 +
6 files changed, 154 insertions(+), 5 deletions(-)
diff --git a/buildstream/__init__.py b/buildstream/__init__.py
index cf56ecf..7748d88 100644
--- a/buildstream/__init__.py
+++ b/buildstream/__init__.py
@@ -33,3 +33,4 @@ if "_BST_COMPLETION" not in os.environ:
from .element import Element, ElementError, Scope
from .buildelement import BuildElement
from .scriptelement import ScriptElement
+ from .sourcetransform import SourceTransform
diff --git a/buildstream/_scheduler/fetchqueue.py b/buildstream/_scheduler/fetchqueue.py
index 24512bd..632b89a 100644
--- a/buildstream/_scheduler/fetchqueue.py
+++ b/buildstream/_scheduler/fetchqueue.py
@@ -39,8 +39,12 @@ class FetchQueue(Queue):
self._skip_cached = skip_cached
def process(self, element):
+ previous_sources = []
for source in element.sources():
- source._fetch()
+ # `previous_sources` is SourceTransform specific and is swallowed
+ # by `**kwargs` in `Source`
+ source._fetch(previous_sources=previous_sources)
+ previous_sources.append(source)
def status(self, element):
# state of dependencies may have changed, recalculate element state
diff --git a/buildstream/element.py b/buildstream/element.py
index fc21f80..ff1b0ce 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -96,6 +96,7 @@ from . import _signals
from . import _site
from ._platform import Platform
from .sandbox._config import SandboxConfig
+from .sourcetransform import SourceTransform
# _KeyStrength():
@@ -1180,6 +1181,10 @@ class Element(Plugin):
# Prepend provenance to the error
raise ElementError("{}: {}".format(self, e), reason=e.reason) from e
+ if self.__sources and isinstance(self.__sources[0], SourceTransform):
+ raise ElementError("{}: A SourceTransform plugin can't be the first source of a build element"
+ .format(self))
+
# Preflight the sources
for source in self.sources():
source._preflight()
@@ -1223,9 +1228,11 @@ class Element(Plugin):
#
def _track(self):
refs = []
- for source in self.__sources:
+ for index, source in enumerate(self.__sources):
old_ref = source.get_ref()
- new_ref = source._track()
+ # `previous_sources` is SourceTransform specific and is swallowed
+ # by `**kwargs` in `Source`
+ new_ref = source._track(previous_sources=self.__sources[0:index])
refs.append((source._get_unique_id(), new_ref))
# Complimentary warning that the new ref will be unused.
diff --git a/buildstream/source.py b/buildstream/source.py
index ec38ae8..124b053 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -373,7 +373,7 @@ class Source(Plugin):
# Wrapper function around plugin provided fetch method
#
- def _fetch(self):
+ def _fetch(self, **kwargs):
self.fetch()
# Wrapper for stage() api which gives the source
@@ -580,7 +580,7 @@ class Source(Plugin):
# Wrapper for track()
#
- def _track(self):
+ def _track(self, **kwargs):
new_ref = self.track()
current_ref = self.get_ref()
diff --git a/buildstream/sourcetransform.py b/buildstream/sourcetransform.py
new file mode 100644
index 0000000..ccf9218
--- /dev/null
+++ b/buildstream/sourcetransform.py
@@ -0,0 +1,136 @@
+#
+# Copyright Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Antoine Wacheux <aw...@bloomberg.net>
+# Gokcen Nurlu <gn...@bloomberg.net>
+"""
+SourceTransform
+===============
+
+
+.. _core_sourcetransform_abstract_methods:
+
+This plugin is a base for package-manager-like source plugins. It can't be the
+first source in an element and it makes use of previously tracked & fetched
+sources.
+
+Abstract Methods
+----------------
+For loading and configuration purposes, SourceTransform based plugins must
+implement the
+:ref:`Plugin base class abstract methods <core_plugin_abstract_methods>` and
+:ref:`Source base class abstract methods <core_source_abstract_methods>`.
+
+Keep in mind that SourceTransform exposes the following abstract methods that
+have function signature different than
+:ref:`their counterparts in Source <core_source_abstract_methods>` and these
+have to be implemented instead.
+
+* :func:`SourceTransform.track(previous_staging_dir) <buildstream.source.SourceTransform.track>`
+
+ Automatically derive a new ref from previously staged sources.
+
+* :func:`SourceTransform.fetch(previous_staging_dir) <buildstream.source.SourceTransform.fetch>`
+
+ Fetch the actual payload for the currently set ref.
+
+"""
+
+from buildstream import Consistency
+from .source import Source
+from . import utils
+from ._exceptions import ImplError
+
+
+class SourceTransform(Source):
+ def __ensure_previous_sources(self, previous_sources):
+ for src in previous_sources:
+ if src.get_consistency() == Consistency.RESOLVED:
+ src._fetch()
+ elif src.get_consistency() == Consistency.INCONSISTENT:
+ new_ref = src._track()
+ src._save_ref(new_ref)
+ src._fetch()
+
+ # Needs explanation
+ def track(self, previous_staging_dir):
+ """Resolve a new ref from the plugin's track option
+
+ Different than :func:`~buildstream.source.Source.track`, implementors
+ have access to previous sources. This one is also mandatory to
+ implement.
+
+ Args:
+ previous_staging_dir (str): Path to a temporary directory where
+ previous sources are staged.
+
+ Returns:
+ (simple object): A new internal source reference, or None
+
+ See :func:`~buildstream.source.Source.get_ref` for a discussion on
+ the *ref* parameter.
+ """
+ raise ImplError("SourceTransform plugin '{}' does not implement track()".format(self.get_kind()))
+
+ # Needs explanation
+ def fetch(self, previous_staging_dir):
+ """Fetch remote sources and mirror them locally, ensuring at least
+ that the specific reference is cached locally.
+
+ Different than :func:`~buildstream.source.Source.fetch`, implementors
+ have access to previous sources.
+
+ Args:
+ previous_staging_dir (str): Path to a temporary directory where
+ previous sources are staged.
+
+ Raises:
+ :class:`.SourceError`
+
+ Implementors should raise :class:`.SourceError` if the there is some
+ network error or if the source reference could not be matched.
+ """
+ raise ImplError("SourceTransform plugin '{}' does not implement fetch()".format(self.get_kind()))
+
+ def _track(self, previous_sources):
+ self.__ensure_previous_sources(previous_sources)
+
+ with utils._tempdir(suffix="tracking") as staging_directory:
+ for src in previous_sources:
+ src._stage(staging_directory)
+
+ # Rest is same with Source._track(), but calling a different .track
+ new_ref = self.track(staging_directory)
+ current_ref = self.get_ref()
+
+ if new_ref is None:
+ # No tracking, keep current ref
+ new_ref = current_ref
+
+ if current_ref != new_ref:
+ self.info("Found new revision: {}".format(new_ref))
+
+ return new_ref
+
+ def _fetch(self, previous_sources):
+ self.__ensure_previous_sources(previous_sources)
+
+ with utils._tempdir(suffix="fetch") as staging_directory:
+ for src in previous_sources:
+ src._stage(staging_directory)
+
+ self.fetch(staging_directory)
diff --git a/doc/source/core_framework.rst b/doc/source/core_framework.rst
index c3b84a9..098df1e 100644
--- a/doc/source/core_framework.rst
+++ b/doc/source/core_framework.rst
@@ -14,6 +14,7 @@ useful for working on BuildStream itself.
buildstream.plugin
buildstream.source
+ buildstream.sourcetransform
buildstream.element
buildstream.buildelement
buildstream.scriptelement