You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:16:39 UTC
[buildstream] branch gokcen/source_transform created (now 60cf4f7)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch gokcen/source_transform
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 60cf4f7 fixup! Add dep_transform example
This branch includes the following new commits:
new 866c4ca Add SourceTransform base class
new ddcc064 Add dep_transform example
new 89b3d79 fixup! Add SourceTransform base class
new 60cf4f7 fixup! Add dep_transform example
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 04/04: fixup! Add dep_transform example
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch gokcen/source_transform
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 60cf4f79d725443ffad47270ff73ed44cb353ad2
Author: Gökçen Nurlu <gn...@bloomberg.net>
AuthorDate: Thu Jun 28 12:02:03 2018 +0100
fixup! Add dep_transform example
---
buildstream/plugins/sources/dep_transform.py | 20 +++++++-------------
1 file changed, 7 insertions(+), 13 deletions(-)
diff --git a/buildstream/plugins/sources/dep_transform.py b/buildstream/plugins/sources/dep_transform.py
index f0d0250..95a3080 100644
--- a/buildstream/plugins/sources/dep_transform.py
+++ b/buildstream/plugins/sources/dep_transform.py
@@ -53,7 +53,7 @@ class DepTransform(SourceTransform):
def configure(self, node):
self.node_validate(node, ['ref'] + Source.COMMON_CONFIG_KEYS)
- self.ref = self.node_get_member(node, str, 'ref', '').strip()
+ self.ref = self.node_get_member(node, str, 'ref', None)
@property
def mirror(self):
@@ -70,12 +70,10 @@ class DepTransform(SourceTransform):
self.host_dep = utils.get_host_tool('dep')
def get_unique_key(self):
- # TODO: This plugin's inputs are actually previous source. What should
- # we do here?
return (self.ref,)
def get_consistency(self):
- if self.ref == '':
+ if self.ref is None:
return Consistency.INCONSISTENT
for dest in (GOPKG_LOCK_FILE, GOPKG_VENDOR_DIR):
dest_path = os.path.join(self.mirror, dest)
@@ -83,7 +81,7 @@ class DepTransform(SourceTransform):
return Consistency.RESOLVED
lock_file_path = os.path.join(self.mirror, GOPKG_LOCK_FILE)
with open(lock_file_path, encoding='utf-8') as lock_file:
- if lock_file.read().strip() == self.ref:
+ if lock_file.read().strip() == self.ref.strip():
return Consistency.CACHED
return Consistency.RESOLVED
@@ -169,20 +167,16 @@ class DepTransform(SourceTransform):
def stage(self, directory):
with self.timed_activity("Staging DepTransform source from based on Gopkg.lock", silent_nested=True):
- shutil.copy(os.path.join(self.mirror, GOPKG_LOCK_FILE),
+ utils.safe_copy(os.path.join(self.mirror, GOPKG_LOCK_FILE),
os.path.join(directory, GOPKG_LOCK_FILE))
# FIXME: Should we ever need to support one, this may
# break on platforms that don't support symlinks if
# a project contains broken symlinks.
target_vendor_dir = os.path.join(directory, GOPKG_VENDOR_DIR)
try:
- shutil.copytree(os.path.join(self.mirror, GOPKG_VENDOR_DIR),
- target_vendor_dir,
- symlinks=True)
- except FileExistsError:
- raise SourceError("{}: Unable to stage vendor directory because it already exists here '{}'"
- .format(self, target_vendor_dir))
- except shutil.Error as err:
+ utils.copy_files(os.path.join(self.mirror, GOPKG_VENDOR_DIR),
+ target_vendor_dir)
+ except utils.UtilError as err:
raise SourceError("{}: Unable to stage vendor directory at '{}'"
.format(self, target_vendor_dir)) from err
[buildstream] 01/04: Add SourceTransform base class
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch gokcen/source_transform
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 866c4ca2291db1b66ab6d48f42ae2bff619c917b
Author: Gökçen Nurlu <gn...@bloomberg.net>
AuthorDate: Thu Jun 7 18:28:58 2018 +0100
Add SourceTransform base class
This adds a new base plugin, an alternative to `Source`, that is more suitable
to implement plugins for languages with package-managers.
---
buildstream/__init__.py | 1 +
buildstream/_scheduler/fetchqueue.py | 6 +-
buildstream/element.py | 11 ++-
buildstream/source.py | 4 +-
buildstream/sourcetransform.py | 136 +++++++++++++++++++++++++++++++++++
doc/source/core_framework.rst | 1 +
6 files changed, 154 insertions(+), 5 deletions(-)
diff --git a/buildstream/__init__.py b/buildstream/__init__.py
index cf56ecf..7748d88 100644
--- a/buildstream/__init__.py
+++ b/buildstream/__init__.py
@@ -33,3 +33,4 @@ if "_BST_COMPLETION" not in os.environ:
from .element import Element, ElementError, Scope
from .buildelement import BuildElement
from .scriptelement import ScriptElement
+ from .sourcetransform import SourceTransform
diff --git a/buildstream/_scheduler/fetchqueue.py b/buildstream/_scheduler/fetchqueue.py
index 24512bd..632b89a 100644
--- a/buildstream/_scheduler/fetchqueue.py
+++ b/buildstream/_scheduler/fetchqueue.py
@@ -39,8 +39,12 @@ class FetchQueue(Queue):
self._skip_cached = skip_cached
def process(self, element):
+ previous_sources = []
for source in element.sources():
- source._fetch()
+ # `previous_sources` is SourceTransform specific and is swallowed
+ # by `**kwargs` in `Source`
+ source._fetch(previous_sources=previous_sources)
+ previous_sources.append(source)
def status(self, element):
# state of dependencies may have changed, recalculate element state
diff --git a/buildstream/element.py b/buildstream/element.py
index fc21f80..ff1b0ce 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -96,6 +96,7 @@ from . import _signals
from . import _site
from ._platform import Platform
from .sandbox._config import SandboxConfig
+from .sourcetransform import SourceTransform
# _KeyStrength():
@@ -1180,6 +1181,10 @@ class Element(Plugin):
# Prepend provenance to the error
raise ElementError("{}: {}".format(self, e), reason=e.reason) from e
+ if self.__sources and isinstance(self.__sources[0], SourceTransform):
+ raise ElementError("{}: A SourceTransform plugin can't be the first source of a build element"
+ .format(self))
+
# Preflight the sources
for source in self.sources():
source._preflight()
@@ -1223,9 +1228,11 @@ class Element(Plugin):
#
def _track(self):
refs = []
- for source in self.__sources:
+ for index, source in enumerate(self.__sources):
old_ref = source.get_ref()
- new_ref = source._track()
+ # `previous_sources` is SourceTransform specific and is swallowed
+ # by `**kwargs` in `Source`
+ new_ref = source._track(previous_sources=self.__sources[0:index])
refs.append((source._get_unique_id(), new_ref))
# Complimentary warning that the new ref will be unused.
diff --git a/buildstream/source.py b/buildstream/source.py
index ec38ae8..124b053 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -373,7 +373,7 @@ class Source(Plugin):
# Wrapper function around plugin provided fetch method
#
- def _fetch(self):
+ def _fetch(self, **kwargs):
self.fetch()
# Wrapper for stage() api which gives the source
@@ -580,7 +580,7 @@ class Source(Plugin):
# Wrapper for track()
#
- def _track(self):
+ def _track(self, **kwargs):
new_ref = self.track()
current_ref = self.get_ref()
diff --git a/buildstream/sourcetransform.py b/buildstream/sourcetransform.py
new file mode 100644
index 0000000..ccf9218
--- /dev/null
+++ b/buildstream/sourcetransform.py
@@ -0,0 +1,136 @@
+#
+# Copyright Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Antoine Wacheux <aw...@bloomberg.net>
+# Gokcen Nurlu <gn...@bloomberg.net>
+"""
+SourceTransform
+===============
+
+
+.. _core_sourcetransform_abstract_methods:
+
+This plugin is a base for package-manager-like source plugins. It can't be the
+first source in an element and it makes use of previously tracked & fetched
+sources.
+
+Abstract Methods
+----------------
+For loading and configuration purposes, SourceTransform based plugins must
+implement the
+:ref:`Plugin base class abstract methods <core_plugin_abstract_methods>` and
+:ref:`Source base class abstract methods <core_source_abstract_methods>`.
+
+Keep in mind that SourceTransform exposes the following abstract methods that
+have function signature different than
+:ref:`their counterparts in Source <core_source_abstract_methods>` and these
+have to be implemented instead.
+
+* :func:`SourceTransform.track(previous_staging_dir) <buildstream.source.SourceTransform.track>`
+
+ Automatically derive a new ref from previously staged sources.
+
+* :func:`SourceTransform.fetch(previous_staging_dir) <buildstream.source.SourceTransform.fetch>`
+
+ Fetch the actual payload for the currently set ref.
+
+"""
+
+from buildstream import Consistency
+from .source import Source
+from . import utils
+from ._exceptions import ImplError
+
+
+class SourceTransform(Source):
+ def __ensure_previous_sources(self, previous_sources):
+ for src in previous_sources:
+ if src.get_consistency() == Consistency.RESOLVED:
+ src._fetch()
+ elif src.get_consistency() == Consistency.INCONSISTENT:
+ new_ref = src._track()
+ src._save_ref(new_ref)
+ src._fetch()
+
+ # Needs explanation
+ def track(self, previous_staging_dir):
+ """Resolve a new ref from the plugin's track option
+
+ Different than :func:`~buildstream.source.Source.track`, implementors
+ have access to previous sources. This one is also mandatory to
+ implement.
+
+ Args:
+ previous_staging_dir (str): Path to a temporary directory where
+ previous sources are staged.
+
+ Returns:
+ (simple object): A new internal source reference, or None
+
+ See :func:`~buildstream.source.Source.get_ref` for a discussion on
+ the *ref* parameter.
+ """
+ raise ImplError("SourceTransform plugin '{}' does not implement track()".format(self.get_kind()))
+
+ # Needs explanation
+ def fetch(self, previous_staging_dir):
+ """Fetch remote sources and mirror them locally, ensuring at least
+ that the specific reference is cached locally.
+
+ Different than :func:`~buildstream.source.Source.fetch`, implementors
+ have access to previous sources.
+
+ Args:
+ previous_staging_dir (str): Path to a temporary directory where
+ previous sources are staged.
+
+ Raises:
+ :class:`.SourceError`
+
+ Implementors should raise :class:`.SourceError` if the there is some
+ network error or if the source reference could not be matched.
+ """
+ raise ImplError("SourceTransform plugin '{}' does not implement fetch()".format(self.get_kind()))
+
+ def _track(self, previous_sources):
+ self.__ensure_previous_sources(previous_sources)
+
+ with utils._tempdir(suffix="tracking") as staging_directory:
+ for src in previous_sources:
+ src._stage(staging_directory)
+
+ # Rest is same with Source._track(), but calling a different .track
+ new_ref = self.track(staging_directory)
+ current_ref = self.get_ref()
+
+ if new_ref is None:
+ # No tracking, keep current ref
+ new_ref = current_ref
+
+ if current_ref != new_ref:
+ self.info("Found new revision: {}".format(new_ref))
+
+ return new_ref
+
+ def _fetch(self, previous_sources):
+ self.__ensure_previous_sources(previous_sources)
+
+ with utils._tempdir(suffix="fetch") as staging_directory:
+ for src in previous_sources:
+ src._stage(staging_directory)
+
+ self.fetch(staging_directory)
diff --git a/doc/source/core_framework.rst b/doc/source/core_framework.rst
index c3b84a9..098df1e 100644
--- a/doc/source/core_framework.rst
+++ b/doc/source/core_framework.rst
@@ -14,6 +14,7 @@ useful for working on BuildStream itself.
buildstream.plugin
buildstream.source
+ buildstream.sourcetransform
buildstream.element
buildstream.buildelement
buildstream.scriptelement
[buildstream] 02/04: Add dep_transform example
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch gokcen/source_transform
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ddcc06456b7814afe7efcdca37f151c39b5a072e
Author: Gökçen Nurlu <gn...@bloomberg.net>
AuthorDate: Mon Jun 18 11:43:07 2018 +0100
Add dep_transform example
---
buildstream/plugins/sources/dep_transform.py | 192 +++++++++++++++++++++++++++
1 file changed, 192 insertions(+)
diff --git a/buildstream/plugins/sources/dep_transform.py b/buildstream/plugins/sources/dep_transform.py
new file mode 100644
index 0000000..f0d0250
--- /dev/null
+++ b/buildstream/plugins/sources/dep_transform.py
@@ -0,0 +1,192 @@
+#
+# Copyright Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Chandan Singh <cs...@bloomberg.net>
+# Gokcen Nurlu <gn...@bloomberg.net>
+"""A SourceTransform implementation for staging Go dependencies via `dep`
+
+**Usage**
+
+.. code: yaml
+ # Specify the godep source kind
+ kind: dep_transform
+
+ # Optionally specify a relative staging directory
+ # directory: path/to/stage
+
+ # Specify the lock file as the ref.
+ # If no ref is provided, the new ref will be the contents of the lock file
+ # generated by running `dep ensure -no-vendor`.
+ ref: "<lock_file content>"
+..
+"""
+
+import errno
+import os
+import shutil
+from hashlib import md5
+
+from buildstream import SourceTransform, Source, SourceError
+from buildstream import utils
+from buildstream import Consistency
+
+GOPKG_LOCK_FILE = 'Gopkg.lock'
+GOPKG_VENDOR_DIR = 'vendor'
+
+
+class DepTransform(SourceTransform):
+ # pylint: disable=attribute-defined-outside-init
+
+ def configure(self, node):
+ self.node_validate(node, ['ref'] + Source.COMMON_CONFIG_KEYS)
+ self.ref = self.node_get_member(node, str, 'ref', '').strip()
+
+ @property
+ def mirror(self):
+ path = os.path.join(
+ self.get_mirror_directory(),
+ self.name,
+ md5(self.ref.encode('utf-8')).hexdigest()
+ )
+ os.makedirs(path, exist_ok=True)
+ return path
+
+ def preflight(self):
+ # Check if dep is installed, get the binaries at the same time
+ self.host_dep = utils.get_host_tool('dep')
+
+ def get_unique_key(self):
+ # TODO: This plugin's inputs are actually previous source. What should
+ # we do here?
+ return (self.ref,)
+
+ def get_consistency(self):
+ if self.ref == '':
+ return Consistency.INCONSISTENT
+ for dest in (GOPKG_LOCK_FILE, GOPKG_VENDOR_DIR):
+ dest_path = os.path.join(self.mirror, dest)
+ if not os.path.exists(dest_path):
+ return Consistency.RESOLVED
+ lock_file_path = os.path.join(self.mirror, GOPKG_LOCK_FILE)
+ with open(lock_file_path, encoding='utf-8') as lock_file:
+ if lock_file.read().strip() == self.ref:
+ return Consistency.CACHED
+ return Consistency.RESOLVED
+
+ def get_ref(self):
+ return self.ref
+
+ def set_ref(self, ref, node):
+ self.ref = node['ref'] = ref
+
+ def track(self, previous_staging_dir):
+ with self.timed_activity('Tracking DepTransform source based on previous sources'):
+ with self.tempdir() as goroot:
+ # dep refuses to work on sources that are not under
+ # GOPATH/src so we need to artificially create that directory
+ # stucture.
+ go_sources_root = os.path.join(goroot, 'src', 'project')
+ os.makedirs(os.path.dirname(go_sources_root), exist_ok=True)
+
+ shutil.move(previous_staging_dir, go_sources_root)
+ gopkg_lock_path = os.path.join(go_sources_root, GOPKG_LOCK_FILE)
+ # Check if the repo has a lock file already
+ if not os.path.isfile(gopkg_lock_path):
+ # If doesn't, let's create a new lock file
+ self.call([self.host_dep, 'ensure', '-no-vendor'],
+ cwd=go_sources_root,
+ env=dict(os.environ, GOPATH=goroot),
+ fail='Failed to update the go dep lock file with the new references')
+
+ with open(gopkg_lock_path, encoding='utf-8') as lock_file:
+ return lock_file.read()
+
+ def fetch(self, previous_staging_dir):
+ with self.timed_activity('Fetching DepTransform source dependencies based on Gopkg.lock'):
+ with self.tempdir() as goroot:
+ # dep refuses to work on sources that are not under
+ # GOPATH/src so we need to artificially create that directory
+ # stucture.
+ go_sources_root = os.path.join(goroot, 'src', 'project')
+ os.makedirs(os.path.dirname(go_sources_root), exist_ok=True)
+
+ shutil.move(previous_staging_dir, go_sources_root)
+
+ gopkg_lock_path = os.path.join(go_sources_root, GOPKG_LOCK_FILE)
+
+ def write_lock_file():
+ with open(gopkg_lock_path, mode='w', encoding='utf-8') as lock_file:
+ lock_file.write(self.get_ref())
+
+ # Is there a lock file present?
+ if os.path.isfile(gopkg_lock_path):
+ # There is! We should ensure it is the same as the ref
+ with open(gopkg_lock_path, encoding='utf-8') as lock_file:
+ stripped_lock_file = lock_file.read().strip()
+ # The current yaml parser strips this already, but let's not rely on it to do so
+ stripped_track_ref = self.get_ref().strip()
+
+ if stripped_lock_file != stripped_track_ref:
+ self.warn("Tracking ref and lock file in source differ. Using tracking ref.")
+ write_lock_file()
+ else:
+ write_lock_file()
+
+ self.call([self.host_dep, 'ensure', '-vendor-only'],
+ cwd=go_sources_root,
+ env=dict(os.environ, GOPATH=goroot),
+ fail='Failed to populate the vendor directory from the lock file')
+
+ # Copy the stuff that we really need
+ for source in (GOPKG_LOCK_FILE, GOPKG_VENDOR_DIR):
+ source_path = os.path.join(go_sources_root, source)
+ dest_path = os.path.join(self.mirror, source)
+ try:
+ os.replace(source_path, dest_path)
+ except OSError as e:
+ # To avoid race condition between two concurrently
+ # running `fetch()` processes.
+ if e.errno not in (errno.ENOTEMPTY, errno.EEXIST):
+ raise
+ self.warn(
+ "{} seems already fetched. ".format(dest_path) +
+ "Continuing with the existing source."
+ )
+
+ def stage(self, directory):
+ with self.timed_activity("Staging DepTransform source from based on Gopkg.lock", silent_nested=True):
+ shutil.copy(os.path.join(self.mirror, GOPKG_LOCK_FILE),
+ os.path.join(directory, GOPKG_LOCK_FILE))
+ # FIXME: Should we ever need to support one, this may
+ # break on platforms that don't support symlinks if
+ # a project contains broken symlinks.
+ target_vendor_dir = os.path.join(directory, GOPKG_VENDOR_DIR)
+ try:
+ shutil.copytree(os.path.join(self.mirror, GOPKG_VENDOR_DIR),
+ target_vendor_dir,
+ symlinks=True)
+ except FileExistsError:
+ raise SourceError("{}: Unable to stage vendor directory because it already exists here '{}'"
+ .format(self, target_vendor_dir))
+ except shutil.Error as err:
+ raise SourceError("{}: Unable to stage vendor directory at '{}'"
+ .format(self, target_vendor_dir)) from err
+
+
+# Plugin entry point
+def setup():
+ return DepTransform
[buildstream] 03/04: fixup! Add SourceTransform base class
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch gokcen/source_transform
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 89b3d794981a00ecdd82378fa4ec007fc50664f2
Author: Gökçen Nurlu <gn...@bloomberg.net>
AuthorDate: Thu Jun 28 11:34:12 2018 +0100
fixup! Add SourceTransform base class
---
buildstream/element.py | 2 +-
buildstream/sourcetransform.py | 6 ++----
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index ff1b0ce..b4856c7 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1182,7 +1182,7 @@ class Element(Plugin):
raise ElementError("{}: {}".format(self, e), reason=e.reason) from e
if self.__sources and isinstance(self.__sources[0], SourceTransform):
- raise ElementError("{}: A SourceTransform plugin can't be the first source of a build element"
+ raise ElementError("{}: A SourceTransform plugin can't be the first source of an element"
.format(self))
# Preflight the sources
diff --git a/buildstream/sourcetransform.py b/buildstream/sourcetransform.py
index ccf9218..e461eaa 100644
--- a/buildstream/sourcetransform.py
+++ b/buildstream/sourcetransform.py
@@ -66,7 +66,6 @@ class SourceTransform(Source):
src._save_ref(new_ref)
src._fetch()
- # Needs explanation
def track(self, previous_staging_dir):
"""Resolve a new ref from the plugin's track option
@@ -86,7 +85,6 @@ class SourceTransform(Source):
"""
raise ImplError("SourceTransform plugin '{}' does not implement track()".format(self.get_kind()))
- # Needs explanation
def fetch(self, previous_staging_dir):
"""Fetch remote sources and mirror them locally, ensuring at least
that the specific reference is cached locally.
@@ -109,7 +107,7 @@ class SourceTransform(Source):
def _track(self, previous_sources):
self.__ensure_previous_sources(previous_sources)
- with utils._tempdir(suffix="tracking") as staging_directory:
+ with self.tempdir() as staging_directory:
for src in previous_sources:
src._stage(staging_directory)
@@ -129,7 +127,7 @@ class SourceTransform(Source):
def _fetch(self, previous_sources):
self.__ensure_previous_sources(previous_sources)
- with utils._tempdir(suffix="fetch") as staging_directory:
+ with self.tempdir() as staging_directory:
for src in previous_sources:
src._stage(staging_directory)