You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ro...@apache.org on 2020/12/29 13:43:00 UTC
[buildstream] branch raoul/892-individual-source-caching created
(now 14c2921)
This is an automated email from the ASF dual-hosted git repository.
root pushed a change to branch raoul/892-individual-source-caching
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 14c2921 Add BST_REQUIRES_PREVIOUS_SOURCE_STAGE option
This branch includes the following new commits:
new 14c2921 Add BST_REQUIRES_PREVIOUS_SOURCE_STAGE option
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/01: Add BST_REQUIRES_PREVIOUS_SOURCE_STAGE option
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch raoul/892-individual-source-caching
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 14c292112975bd0e890886f4b3466feaf3fc7823
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Mon Apr 29 17:36:52 2019 +0100
Add BST_REQUIRES_PREVIOUS_SOURCE_STAGE option
This is an element option that allows sources to be staged more
seperately where possible rather than on a per element option. When
there is a source that has this option set to true, all previously
listed sources will be cached with this, and sources listed afterwards
will be cached individually. If no sources have this option, all sources
will be cached seperately.
Fixes #892
---
buildstream/element.py | 89 ++++++++++++++++------
buildstream/plugins/sources/patch.py | 2 +
buildstream/source.py | 14 +++-
.../plugins/sources/foo_transform.py | 1 +
4 files changed, 82 insertions(+), 24 deletions(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index 05884c0..e897992 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -107,6 +107,7 @@ from ._artifact import Artifact
from .storage.directory import Directory
from .storage._filebaseddirectory import FileBasedDirectory
+from .storage._casbaseddirectory import CasBasedDirectory
from .storage.directory import VirtualDirectoryError
@@ -1537,9 +1538,17 @@ class Element(Plugin):
if self.__sources:
- sourcecache = self._get_context().sourcecache
+ sourcecache = context.sourcecache
+ # find last required source
+ last_required_previous_ix = 0
+ for ix, source in enumerate(self.__sources):
+ if source.BST_REQUIRES_PREVIOUS_SOURCES_CACHE:
+ last_required_previous_ix = ix
+
try:
- import_dir = sourcecache.export(self.__sources[-1])
+ import_dir = CasBasedDirectory(context.get_cascache())
+ for source in self.__sources[last_required_previous_ix:]:
+ import_dir.import_files(sourcecache.export(source))
except SourceCacheError as e:
raise ElementError("Error trying to export source for {}: {}"
.format(self.name, e))
@@ -2192,24 +2201,32 @@ class Element(Plugin):
def _fetch(self, fetch_original=False):
previous_sources = []
sources = self.__sources
+ fetch_needed = False
if sources and not fetch_original:
- source = sources[-1]
- if self.__sourcecache.contains(source):
- return
+ last_requires_previous = 0
+ for ix, source in enumerate(self.__sources):
+ if source.BST_REQUIRES_PREVIOUS_SOURCES_CACHE:
+ last_requires_previous = ix
- # try and fetch from source cache
- if source._get_consistency() < Consistency.CACHED and \
- self.__sourcecache.has_fetch_remotes() and \
- not self.__sourcecache.contains(source):
- if self.__sourcecache.pull(source):
- return
+ for source in self.__sources[last_requires_previous:]:
+ if self.__sourcecache.contains(source):
+ continue
+
+ # try and fetch from source cache
+ if source._get_consistency() < Consistency.CACHED and \
+ self.__sourcecache.has_fetch_remotes():
+ if self.__sourcecache.pull(source):
+ continue
+
+ fetch_needed = True
# We need to fetch original sources
- for source in self.sources():
- source_consistency = source._get_consistency()
- if source_consistency != Consistency.CACHED:
- source._fetch(previous_sources)
- previous_sources.append(source)
+ if fetch_needed or fetch_original:
+ for source in self.sources():
+ source_consistency = source._get_consistency()
+ if source_consistency != Consistency.CACHED:
+ source._fetch(previous_sources)
+ previous_sources.append(source)
self.__cache_sources()
@@ -2264,12 +2281,27 @@ class Element(Plugin):
# Check if sources are cached, generating the source key if it hasn't been
def _source_cached(self):
if self.__sources:
- last_source = self.__sources[-1]
- if not last_source._key:
- last_source._generate_key(self.__sources[:-1])
- return self._get_context().sourcecache.contains(last_source)
- else:
- return True
+ last_requires_previous = 0
+ for ix, source in enumerate(self.__sources):
+ if source.BST_REQUIRES_PREVIOUS_SOURCES_CACHE:
+ last_requires_previous = ix
+
+ sourcecache = self._get_context().sourcecache
+ if not self.__sources[last_requires_previous]._key:
+ self.__sources[last_requires_previous]._generate_key(
+ self.__sources[:last_requires_previous])
+
+ # Go through individual sources
+ for source in self.__sources[last_requires_previous+1:]:
+ if not source._key:
+ source._generate_key([])
+ if not sourcecache.contains(source):
+ return False
+
+ if not sourcecache.contains(self.__sources[last_requires_previous]):
+ return False
+
+ return True
def _should_fetch(self, fetch_original=False):
""" return bool of if we need to run the fetch stage for this element
@@ -2918,7 +2950,18 @@ class Element(Plugin):
def __cache_sources(self):
sources = self.__sources
if sources and not self._source_cached():
- sources[-1]._cache(sources[:-1])
+ last_requires_previous = 0
+ for ix, source in enumerate(sources):
+ if source.BST_REQUIRES_PREVIOUS_SOURCES_CACHE:
+ last_requires_previous = ix
+
+ # cache last source that requires previous sources
+ self.__sourcecache.commit(sources[last_requires_previous],
+ sources[:last_requires_previous])
+
+ # commit all other sources by themselves
+ for source in sources[last_requires_previous+1:]:
+ self.__sourcecache.commit(source, [])
# __update_state_recursively()
#
diff --git a/buildstream/plugins/sources/patch.py b/buildstream/plugins/sources/patch.py
index 8e833b4..6da1ea0 100644
--- a/buildstream/plugins/sources/patch.py
+++ b/buildstream/plugins/sources/patch.py
@@ -52,6 +52,8 @@ from buildstream import utils
class PatchSource(Source):
# pylint: disable=attribute-defined-outside-init
+ BST_REQUIRES_PREVIOUS_SOURCES_CACHE = True
+
def configure(self, node):
self.path = self.node_get_project_path(node, 'path',
check_is_file=True)
diff --git a/buildstream/source.py b/buildstream/source.py
index 6f4ff57..94cb82e 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -286,6 +286,15 @@ class Source(Plugin):
*Since: 1.4*
"""
+ BST_REQUIRES_PREVIOUS_SOURCES_CACHE = False
+ """Whether access to previous sources is required during cache
+
+ When set to True:
+ * all sources listed before current source in the given element will be
+ passed to the source when it's cached.
+ * This source can not be the first source for an element.
+ """
+
def __init__(self, context, project, meta, *, alias_override=None):
provenance = _yaml.node_get_provenance(meta.config)
super().__init__("{}-{}".format(meta.element_name, meta.element_index),
@@ -709,7 +718,10 @@ class Source(Plugin):
def _cache(self, previous_sources):
# stage the source into the source cache
- self.__source_cache.commit(self, previous_sources)
+ if self.BST_REQUIRES_PREVIOUS_SOURCES_CACHE:
+ self.__source_cache.commit(self, previous_sources)
+ else:
+ self.__source_cache.commit(self, [])
# Wrapper for stage() api which gives the source
# plugin a fully constructed path considering the
diff --git a/tests/sources/previous_source_access/plugins/sources/foo_transform.py b/tests/sources/previous_source_access/plugins/sources/foo_transform.py
index 8209464..bec4f99 100644
--- a/tests/sources/previous_source_access/plugins/sources/foo_transform.py
+++ b/tests/sources/previous_source_access/plugins/sources/foo_transform.py
@@ -18,6 +18,7 @@ class FooTransformSource(Source):
# We need access to previous both at track time and fetch time
BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True
BST_REQUIRES_PREVIOUS_SOURCES_FETCH = True
+ BST_REQUIRES_PREVIOUS_SOURCES_CACHE = True
@property
def mirror(self):