You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:57:00 UTC
[buildstream] 01/02: Add support for importing dependencies in
sysroots.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch valentdavid/sysroot_dependencies
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 85b65dc7ff194f4cae471e66a2e338f8be42a178
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Wed Oct 3 19:42:35 2018 +0200
Add support for importing dependencies in sysroots.
This feature is useful when bootstraping a system.
---
buildstream/_loader/loadelement.py | 8 +++
buildstream/_loader/loader.py | 28 +++++++--
buildstream/_loader/metaelement.py | 2 +
buildstream/element.py | 114 +++++++++++++++++++++++++++----------
4 files changed, 117 insertions(+), 35 deletions(-)
diff --git a/buildstream/_loader/loadelement.py b/buildstream/_loader/loadelement.py
index 4104dfd..68f9dad 100644
--- a/buildstream/_loader/loadelement.py
+++ b/buildstream/_loader/loadelement.py
@@ -72,8 +72,16 @@ class LoadElement():
'variables', 'environment', 'environment-nocache',
'config', 'public', 'description',
'build-depends', 'runtime-depends',
+ 'sysroots',
])
+ self.sysroots = []
+ sysroots = _yaml.node_get(node, list, 'sysroots', default_value=[])
+ for sysroot in sysroots:
+ _yaml.node_validate(sysroot, ['path', 'depends', 'build-depends', 'runtime-depends'])
+ path = _yaml.node_get(sysroot, str, 'path')
+ self.sysroots.append((path, _extract_depends_from_node(sysroot)))
+
# Extract the Dependencies
self.deps = _extract_depends_from_node(self.node)
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index 1b27d9d..2335cb9 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -36,6 +36,14 @@ from . import MetaElement
from . import MetaSource
+def _dependencies(element):
+ for dep in element.deps:
+ yield dep
+ for path, deps in element.sysroots:
+ for dep in deps:
+ yield dep
+
+
# Loader():
#
# The Loader class does the heavy lifting of parsing target
@@ -122,9 +130,9 @@ class Loader():
# Set up a dummy element that depends on all top-level targets
# to resolve potential circular dependencies between them
- DummyTarget = namedtuple('DummyTarget', ['name', 'full_name', 'deps'])
+ DummyTarget = namedtuple('DummyTarget', ['name', 'full_name', 'deps', 'sysroots'])
- dummy = DummyTarget(name='', full_name='', deps=deps)
+ dummy = DummyTarget(name='', full_name='', deps=deps, sysroots=[])
self._elements[''] = dummy
profile_key = "_".join(t for t in targets)
@@ -259,7 +267,7 @@ class Loader():
self._elements[filename] = element
# Load all dependency files for the new LoadElement
- for dep in element.deps:
+ for dep in _dependencies(element):
if dep.junction:
self._load_file(dep.junction, rewritable, ticker, fetch_subprojects)
loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
@@ -276,6 +284,7 @@ class Loader():
return element
+
# _check_circular_deps():
#
# Detect circular dependencies on LoadElements with
@@ -311,7 +320,7 @@ class Loader():
# Push / Check each dependency / Pop
check_elements[element_name] = True
- for dep in element.deps:
+ for dep in _dependencies(element):
loader = self._get_loader_for_dep(dep)
loader._check_circular_deps(dep.name, check_elements, validated)
del check_elements[element_name]
@@ -392,6 +401,8 @@ class Loader():
# directly or indirectly depends on another direct dependency,
# it is found later in the list.
element.deps.sort(key=cmp_to_key(dependency_cmp))
+ for path, deps in element.sysroots:
+ deps.sort(key=cmp_to_key(dependency_cmp))
visited[element_name] = True
@@ -459,6 +470,15 @@ class Loader():
if dep.dep_type != 'build':
meta_element.dependencies.append(meta_dep)
+ for path, deps in element.sysroots:
+ for dep in deps:
+ loader = self._get_loader_for_dep(dep)
+ meta_dep = loader._collect_element(dep.name)
+ if dep.dep_type != 'runtime':
+ meta_element.sysroot_build_dependencies.append((path, meta_dep))
+ if dep.dep_type != 'build':
+ meta_element.sysroot_dependencies.append((path, meta_dep))
+
return meta_element
# _get_loader():
diff --git a/buildstream/_loader/metaelement.py b/buildstream/_loader/metaelement.py
index c13d559..2a1a897 100644
--- a/buildstream/_loader/metaelement.py
+++ b/buildstream/_loader/metaelement.py
@@ -54,4 +54,6 @@ class MetaElement():
self.sandbox = sandbox
self.build_dependencies = []
self.dependencies = []
+ self.sysroot_build_dependencies = []
+ self.sysroot_dependencies = []
self.first_pass = first_pass
diff --git a/buildstream/element.py b/buildstream/element.py
index 320ba7a..331e329 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -195,6 +195,7 @@ class Element(Plugin):
self.__runtime_dependencies = [] # Direct runtime dependency Elements
self.__build_dependencies = [] # Direct build dependency Elements
+ self.__sysroots = {}
self.__sources = [] # List of Sources
self.__weak_cache_key = None # Our cached weak cache key
self.__strict_cache_key = None # Our cached cache key for strict builds
@@ -374,7 +375,9 @@ class Element(Plugin):
for source in self.__sources:
yield source
- def dependencies(self, scope, *, recurse=True, visited=None, recursed=False):
+
+ def dependencies(self, scope, *, recurse=True, visited=None, recursed=False,
+ with_sysroot=False, sysroot='/'):
"""dependencies(scope, *, recurse=True)
A generator function which yields the dependencies of the given element.
@@ -399,40 +402,75 @@ class Element(Plugin):
scope_set = set((Scope.BUILD, Scope.RUN)) if scope == Scope.ALL else set((scope,))
- if full_name in visited and scope_set.issubset(visited[full_name]):
+ if (sysroot, full_name) in visited and scope_set.issubset(visited[(sysroot, full_name)]):
return
should_yield = False
if full_name not in visited:
- visited[full_name] = scope_set
+ visited[(sysroot, full_name)] = scope_set
should_yield = True
else:
- visited[full_name] |= scope_set
+ visited[(sysroot, full_name)] |= scope_set
if recurse or not recursed:
if scope == Scope.ALL:
for dep in self.__build_dependencies:
yield from dep.dependencies(Scope.ALL, recurse=recurse,
- visited=visited, recursed=True)
+ visited=visited, recursed=True,
+ sysroot=sysroot, with_sysroot=with_sysroot)
for dep in self.__runtime_dependencies:
if dep not in self.__build_dependencies:
yield from dep.dependencies(Scope.ALL, recurse=recurse,
- visited=visited, recursed=True)
+ visited=visited, recursed=True,
+ sysroot=sysroot, with_sysroot=with_sysroot)
+
+ for path, value in self.__sysroots.items():
+ new_sysroot = path if not recursed else sysroot
+ run_deps, build_deps = value
+ for dep in build_deps:
+ yield from dep.dependencies(Scope.ALL, recurse=recurse,
+ visited=visited, recursed=True,
+ sysroot=new_sysroot, with_sysroot=with_sysroot)
+ for dep in run_deps:
+ if dep not in build_deps:
+ yield from dep.dependencies(Scope.ALL, recurse=recurse,
+ visited=visited, recursed=True,
+ sysroot=new_sysroot, with_sysroot=with_sysroot)
elif scope == Scope.BUILD:
for dep in self.__build_dependencies:
yield from dep.dependencies(Scope.RUN, recurse=recurse,
- visited=visited, recursed=True)
+ visited=visited, recursed=True,
+ sysroot=sysroot, with_sysroot=with_sysroot)
+ for path, value in self.__sysroots.items():
+ new_sysroot = path if not recursed else sysroot
+ run_deps, build_deps = value
+ for dep in build_deps:
+ yield from dep.dependencies(Scope.RUN, recurse=recurse,
+ visited=visited, recursed=True,
+ sysroot=new_sysroot, with_sysroot=with_sysroot)
elif scope == Scope.RUN:
for dep in self.__runtime_dependencies:
yield from dep.dependencies(Scope.RUN, recurse=recurse,
- visited=visited, recursed=True)
+ visited=visited, recursed=True,
+ sysroot=sysroot, with_sysroot=with_sysroot)
+ for path, value in self.__sysroots.items():
+ new_sysroot = path if not recursed else sysroot
+ run_deps, build_deps = value
+ for dep in run_deps:
+ yield from dep.dependencies(Scope.RUN, recurse=recurse,
+ visited=visited, recursed=True,
+ sysroot=new_sysroot, with_sysroot=with_sysroot)
# Yeild self only at the end, after anything needed has been traversed
if should_yield and (recurse or recursed) and (scope == Scope.ALL or scope == Scope.RUN):
- yield self
+ if with_sysroot:
+ yield sysroot, self
+ else:
+ yield self
+
def search(self, scope, name):
"""Search for a dependency by name
@@ -631,7 +669,7 @@ class Element(Plugin):
vbasedir = sandbox.get_virtual_directory()
vstagedir = vbasedir \
if path is None \
- else vbasedir.descend(path.lstrip(os.sep).split(os.sep))
+ else vbasedir.descend(path.lstrip(os.sep).split(os.sep), create=True)
files = list(self.__compute_splits(include, exclude, orphans))
@@ -649,7 +687,8 @@ class Element(Plugin):
return link_result.combine(copy_result)
def stage_dependency_artifacts(self, sandbox, scope, *, path=None,
- include=None, exclude=None, orphans=True):
+ include=None, exclude=None, orphans=True,
+ build=True):
"""Stage element dependencies in scope
This is primarily a convenience wrapper around
@@ -679,7 +718,7 @@ class Element(Plugin):
if self.__can_build_incrementally() and workspace.last_successful:
old_dep_keys = self.__get_artifact_metadata_dependencies(workspace.last_successful)
- for dep in self.dependencies(scope):
+ for sysroot, dep in self.dependencies(scope, with_sysroot=True):
# If we are workspaced, and we therefore perform an
# incremental build, we must ensure that we update the mtimes
# of any files created by our dependencies since the last
@@ -704,8 +743,13 @@ class Element(Plugin):
if utils._is_main_process():
self._get_context().get_workspaces().save_config()
+ if build:
+ sub_path = os.path.join(path, os.path.relpath(sysroot, '/')) if path else sysroot
+ else:
+ sub_path = path
+
result = dep.stage_artifact(sandbox,
- path=path,
+ path=sub_path,
include=include,
exclude=exclude,
orphans=orphans,
@@ -909,6 +953,17 @@ class Element(Plugin):
dependency = Element._new_from_meta(meta_dep, artifacts)
element.__build_dependencies.append(dependency)
+ for path, meta_dep in meta.sysroot_dependencies:
+ if path not in element.__sysroots:
+ element.__sysroots[path] = ([], [])
+ dependency = Element._new_from_meta(meta_dep, artifacts)
+ element.__sysroots[path][0].append(dependency)
+ for path, meta_dep in meta.sysroot_build_dependencies:
+ if path not in element.__sysroots:
+ element.__sysroots[path] = ([], [])
+ dependency = Element._new_from_meta(meta_dep, artifacts)
+ element.__sysroots[path][1].append(dependency)
+
return element
# _get_redundant_source_refs()
@@ -1086,17 +1141,17 @@ class Element(Plugin):
# Calculate weak cache key
# Weak cache key includes names of direct build dependencies
# but does not include keys of dependencies.
+ dependencies = []
if self.BST_STRICT_REBUILD:
- dependencies = [
- e._get_cache_key(strength=_KeyStrength.WEAK)
- for e in self.dependencies(Scope.BUILD)
- ]
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True):
+ dependencies = [(sysroot, e._get_cache_key(strength=_KeyStrength.WEAK))
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
else:
- dependencies = [
- e.name for e in self.dependencies(Scope.BUILD, recurse=False)
- ]
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True):
+ dependencies = [(sysroot, e.name)
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
- self.__weak_cache_key = self.__calculate_cache_key(dependencies)
+ self.__weak_cache_key = self.__calculate_cache_key(sorted(dependencies))
if self.__weak_cache_key is None:
# Weak cache key could not be calculated yet
@@ -1122,10 +1177,9 @@ class Element(Plugin):
return
if self.__strict_cache_key is None:
- dependencies = [
- e.__strict_cache_key for e in self.dependencies(Scope.BUILD)
- ]
- self.__strict_cache_key = self.__calculate_cache_key(dependencies)
+ dependencies = [(sysroot, e.__strict_cache_key)
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
+ self.__strict_cache_key = self.__calculate_cache_key(sorted(dependencies))
if self.__strict_cache_key is None:
# Strict cache key could not be calculated yet
@@ -1164,11 +1218,9 @@ class Element(Plugin):
strong_key, _ = self.__get_artifact_metadata_keys()
self.__cache_key = strong_key
elif self.__assemble_scheduled or self.__assemble_done:
- # Artifact will or has been built, not downloaded
- dependencies = [
- e._get_cache_key() for e in self.dependencies(Scope.BUILD)
- ]
- self.__cache_key = self.__calculate_cache_key(dependencies)
+ dependencies = [(sysroot, e._get_cache_key())
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
+ self.__cache_key = self.__calculate_cache_key(sorted(dependencies))
if self.__cache_key is None:
# Strong cache key could not be calculated yet
@@ -1329,7 +1381,7 @@ class Element(Plugin):
# Stage deps in the sandbox root
if deps == 'run':
with self.timed_activity("Staging dependencies", silent_nested=True):
- self.stage_dependency_artifacts(sandbox, scope)
+ self.stage_dependency_artifacts(sandbox, scope, build=False)
# Run any integration commands provided by the dependencies
# once they are all staged and ready