You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:56:59 UTC
[buildstream] branch valentdavid/sysroot_dependencies created (now
90a370a)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch valentdavid/sysroot_dependencies
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 90a370a Fix key sorting when using workspaces
This branch includes the following new commits:
new 85b65dc Add support for importing dependencies in sysroots.
new 90a370a Fix key sorting when using workspaces
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/02: Add support for importing dependencies in
sysroots.
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch valentdavid/sysroot_dependencies
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 85b65dc7ff194f4cae471e66a2e338f8be42a178
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Wed Oct 3 19:42:35 2018 +0200
Add support for importing dependencies in sysroots.
This feature is useful when bootstraping a system.
---
buildstream/_loader/loadelement.py | 8 +++
buildstream/_loader/loader.py | 28 +++++++--
buildstream/_loader/metaelement.py | 2 +
buildstream/element.py | 114 +++++++++++++++++++++++++++----------
4 files changed, 117 insertions(+), 35 deletions(-)
diff --git a/buildstream/_loader/loadelement.py b/buildstream/_loader/loadelement.py
index 4104dfd..68f9dad 100644
--- a/buildstream/_loader/loadelement.py
+++ b/buildstream/_loader/loadelement.py
@@ -72,8 +72,16 @@ class LoadElement():
'variables', 'environment', 'environment-nocache',
'config', 'public', 'description',
'build-depends', 'runtime-depends',
+ 'sysroots',
])
+ self.sysroots = []
+ sysroots = _yaml.node_get(node, list, 'sysroots', default_value=[])
+ for sysroot in sysroots:
+ _yaml.node_validate(sysroot, ['path', 'depends', 'build-depends', 'runtime-depends'])
+ path = _yaml.node_get(sysroot, str, 'path')
+ self.sysroots.append((path, _extract_depends_from_node(sysroot)))
+
# Extract the Dependencies
self.deps = _extract_depends_from_node(self.node)
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index 1b27d9d..2335cb9 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -36,6 +36,14 @@ from . import MetaElement
from . import MetaSource
+def _dependencies(element):
+ for dep in element.deps:
+ yield dep
+ for path, deps in element.sysroots:
+ for dep in deps:
+ yield dep
+
+
# Loader():
#
# The Loader class does the heavy lifting of parsing target
@@ -122,9 +130,9 @@ class Loader():
# Set up a dummy element that depends on all top-level targets
# to resolve potential circular dependencies between them
- DummyTarget = namedtuple('DummyTarget', ['name', 'full_name', 'deps'])
+ DummyTarget = namedtuple('DummyTarget', ['name', 'full_name', 'deps', 'sysroots'])
- dummy = DummyTarget(name='', full_name='', deps=deps)
+ dummy = DummyTarget(name='', full_name='', deps=deps, sysroots=[])
self._elements[''] = dummy
profile_key = "_".join(t for t in targets)
@@ -259,7 +267,7 @@ class Loader():
self._elements[filename] = element
# Load all dependency files for the new LoadElement
- for dep in element.deps:
+ for dep in _dependencies(element):
if dep.junction:
self._load_file(dep.junction, rewritable, ticker, fetch_subprojects)
loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
@@ -276,6 +284,7 @@ class Loader():
return element
+
# _check_circular_deps():
#
# Detect circular dependencies on LoadElements with
@@ -311,7 +320,7 @@ class Loader():
# Push / Check each dependency / Pop
check_elements[element_name] = True
- for dep in element.deps:
+ for dep in _dependencies(element):
loader = self._get_loader_for_dep(dep)
loader._check_circular_deps(dep.name, check_elements, validated)
del check_elements[element_name]
@@ -392,6 +401,8 @@ class Loader():
# directly or indirectly depends on another direct dependency,
# it is found later in the list.
element.deps.sort(key=cmp_to_key(dependency_cmp))
+ for path, deps in element.sysroots:
+ deps.sort(key=cmp_to_key(dependency_cmp))
visited[element_name] = True
@@ -459,6 +470,15 @@ class Loader():
if dep.dep_type != 'build':
meta_element.dependencies.append(meta_dep)
+ for path, deps in element.sysroots:
+ for dep in deps:
+ loader = self._get_loader_for_dep(dep)
+ meta_dep = loader._collect_element(dep.name)
+ if dep.dep_type != 'runtime':
+ meta_element.sysroot_build_dependencies.append((path, meta_dep))
+ if dep.dep_type != 'build':
+ meta_element.sysroot_dependencies.append((path, meta_dep))
+
return meta_element
# _get_loader():
diff --git a/buildstream/_loader/metaelement.py b/buildstream/_loader/metaelement.py
index c13d559..2a1a897 100644
--- a/buildstream/_loader/metaelement.py
+++ b/buildstream/_loader/metaelement.py
@@ -54,4 +54,6 @@ class MetaElement():
self.sandbox = sandbox
self.build_dependencies = []
self.dependencies = []
+ self.sysroot_build_dependencies = []
+ self.sysroot_dependencies = []
self.first_pass = first_pass
diff --git a/buildstream/element.py b/buildstream/element.py
index 320ba7a..331e329 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -195,6 +195,7 @@ class Element(Plugin):
self.__runtime_dependencies = [] # Direct runtime dependency Elements
self.__build_dependencies = [] # Direct build dependency Elements
+ self.__sysroots = {}
self.__sources = [] # List of Sources
self.__weak_cache_key = None # Our cached weak cache key
self.__strict_cache_key = None # Our cached cache key for strict builds
@@ -374,7 +375,9 @@ class Element(Plugin):
for source in self.__sources:
yield source
- def dependencies(self, scope, *, recurse=True, visited=None, recursed=False):
+
+ def dependencies(self, scope, *, recurse=True, visited=None, recursed=False,
+ with_sysroot=False, sysroot='/'):
"""dependencies(scope, *, recurse=True)
A generator function which yields the dependencies of the given element.
@@ -399,40 +402,75 @@ class Element(Plugin):
scope_set = set((Scope.BUILD, Scope.RUN)) if scope == Scope.ALL else set((scope,))
- if full_name in visited and scope_set.issubset(visited[full_name]):
+ if (sysroot, full_name) in visited and scope_set.issubset(visited[(sysroot, full_name)]):
return
should_yield = False
if full_name not in visited:
- visited[full_name] = scope_set
+ visited[(sysroot, full_name)] = scope_set
should_yield = True
else:
- visited[full_name] |= scope_set
+ visited[(sysroot, full_name)] |= scope_set
if recurse or not recursed:
if scope == Scope.ALL:
for dep in self.__build_dependencies:
yield from dep.dependencies(Scope.ALL, recurse=recurse,
- visited=visited, recursed=True)
+ visited=visited, recursed=True,
+ sysroot=sysroot, with_sysroot=with_sysroot)
for dep in self.__runtime_dependencies:
if dep not in self.__build_dependencies:
yield from dep.dependencies(Scope.ALL, recurse=recurse,
- visited=visited, recursed=True)
+ visited=visited, recursed=True,
+ sysroot=sysroot, with_sysroot=with_sysroot)
+
+ for path, value in self.__sysroots.items():
+ new_sysroot = path if not recursed else sysroot
+ run_deps, build_deps = value
+ for dep in build_deps:
+ yield from dep.dependencies(Scope.ALL, recurse=recurse,
+ visited=visited, recursed=True,
+ sysroot=new_sysroot, with_sysroot=with_sysroot)
+ for dep in run_deps:
+ if dep not in build_deps:
+ yield from dep.dependencies(Scope.ALL, recurse=recurse,
+ visited=visited, recursed=True,
+ sysroot=new_sysroot, with_sysroot=with_sysroot)
elif scope == Scope.BUILD:
for dep in self.__build_dependencies:
yield from dep.dependencies(Scope.RUN, recurse=recurse,
- visited=visited, recursed=True)
+ visited=visited, recursed=True,
+ sysroot=sysroot, with_sysroot=with_sysroot)
+ for path, value in self.__sysroots.items():
+ new_sysroot = path if not recursed else sysroot
+ run_deps, build_deps = value
+ for dep in build_deps:
+ yield from dep.dependencies(Scope.RUN, recurse=recurse,
+ visited=visited, recursed=True,
+ sysroot=new_sysroot, with_sysroot=with_sysroot)
elif scope == Scope.RUN:
for dep in self.__runtime_dependencies:
yield from dep.dependencies(Scope.RUN, recurse=recurse,
- visited=visited, recursed=True)
+ visited=visited, recursed=True,
+ sysroot=sysroot, with_sysroot=with_sysroot)
+ for path, value in self.__sysroots.items():
+ new_sysroot = path if not recursed else sysroot
+ run_deps, build_deps = value
+ for dep in run_deps:
+ yield from dep.dependencies(Scope.RUN, recurse=recurse,
+ visited=visited, recursed=True,
+ sysroot=new_sysroot, with_sysroot=with_sysroot)
# Yeild self only at the end, after anything needed has been traversed
if should_yield and (recurse or recursed) and (scope == Scope.ALL or scope == Scope.RUN):
- yield self
+ if with_sysroot:
+ yield sysroot, self
+ else:
+ yield self
+
def search(self, scope, name):
"""Search for a dependency by name
@@ -631,7 +669,7 @@ class Element(Plugin):
vbasedir = sandbox.get_virtual_directory()
vstagedir = vbasedir \
if path is None \
- else vbasedir.descend(path.lstrip(os.sep).split(os.sep))
+ else vbasedir.descend(path.lstrip(os.sep).split(os.sep), create=True)
files = list(self.__compute_splits(include, exclude, orphans))
@@ -649,7 +687,8 @@ class Element(Plugin):
return link_result.combine(copy_result)
def stage_dependency_artifacts(self, sandbox, scope, *, path=None,
- include=None, exclude=None, orphans=True):
+ include=None, exclude=None, orphans=True,
+ build=True):
"""Stage element dependencies in scope
This is primarily a convenience wrapper around
@@ -679,7 +718,7 @@ class Element(Plugin):
if self.__can_build_incrementally() and workspace.last_successful:
old_dep_keys = self.__get_artifact_metadata_dependencies(workspace.last_successful)
- for dep in self.dependencies(scope):
+ for sysroot, dep in self.dependencies(scope, with_sysroot=True):
# If we are workspaced, and we therefore perform an
# incremental build, we must ensure that we update the mtimes
# of any files created by our dependencies since the last
@@ -704,8 +743,13 @@ class Element(Plugin):
if utils._is_main_process():
self._get_context().get_workspaces().save_config()
+ if build:
+ sub_path = os.path.join(path, os.path.relpath(sysroot, '/')) if path else sysroot
+ else:
+ sub_path = path
+
result = dep.stage_artifact(sandbox,
- path=path,
+ path=sub_path,
include=include,
exclude=exclude,
orphans=orphans,
@@ -909,6 +953,17 @@ class Element(Plugin):
dependency = Element._new_from_meta(meta_dep, artifacts)
element.__build_dependencies.append(dependency)
+ for path, meta_dep in meta.sysroot_dependencies:
+ if path not in element.__sysroots:
+ element.__sysroots[path] = ([], [])
+ dependency = Element._new_from_meta(meta_dep, artifacts)
+ element.__sysroots[path][0].append(dependency)
+ for path, meta_dep in meta.sysroot_build_dependencies:
+ if path not in element.__sysroots:
+ element.__sysroots[path] = ([], [])
+ dependency = Element._new_from_meta(meta_dep, artifacts)
+ element.__sysroots[path][1].append(dependency)
+
return element
# _get_redundant_source_refs()
@@ -1086,17 +1141,17 @@ class Element(Plugin):
# Calculate weak cache key
# Weak cache key includes names of direct build dependencies
# but does not include keys of dependencies.
+ dependencies = []
if self.BST_STRICT_REBUILD:
- dependencies = [
- e._get_cache_key(strength=_KeyStrength.WEAK)
- for e in self.dependencies(Scope.BUILD)
- ]
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True):
+ dependencies = [(sysroot, e._get_cache_key(strength=_KeyStrength.WEAK))
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
else:
- dependencies = [
- e.name for e in self.dependencies(Scope.BUILD, recurse=False)
- ]
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True):
+ dependencies = [(sysroot, e.name)
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
- self.__weak_cache_key = self.__calculate_cache_key(dependencies)
+ self.__weak_cache_key = self.__calculate_cache_key(sorted(dependencies))
if self.__weak_cache_key is None:
# Weak cache key could not be calculated yet
@@ -1122,10 +1177,9 @@ class Element(Plugin):
return
if self.__strict_cache_key is None:
- dependencies = [
- e.__strict_cache_key for e in self.dependencies(Scope.BUILD)
- ]
- self.__strict_cache_key = self.__calculate_cache_key(dependencies)
+ dependencies = [(sysroot, e.__strict_cache_key)
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
+ self.__strict_cache_key = self.__calculate_cache_key(sorted(dependencies))
if self.__strict_cache_key is None:
# Strict cache key could not be calculated yet
@@ -1164,11 +1218,9 @@ class Element(Plugin):
strong_key, _ = self.__get_artifact_metadata_keys()
self.__cache_key = strong_key
elif self.__assemble_scheduled or self.__assemble_done:
- # Artifact will or has been built, not downloaded
- dependencies = [
- e._get_cache_key() for e in self.dependencies(Scope.BUILD)
- ]
- self.__cache_key = self.__calculate_cache_key(dependencies)
+ dependencies = [(sysroot, e._get_cache_key())
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
+ self.__cache_key = self.__calculate_cache_key(sorted(dependencies))
if self.__cache_key is None:
# Strong cache key could not be calculated yet
@@ -1329,7 +1381,7 @@ class Element(Plugin):
# Stage deps in the sandbox root
if deps == 'run':
with self.timed_activity("Staging dependencies", silent_nested=True):
- self.stage_dependency_artifacts(sandbox, scope)
+ self.stage_dependency_artifacts(sandbox, scope, build=False)
# Run any integration commands provided by the dependencies
# once they are all staged and ready
[buildstream] 02/02: Fix key sorting when using workspaces
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch valentdavid/sysroot_dependencies
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 90a370ae63bcda01ad705a018b9c029d4373c290
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Tue Oct 16 12:17:43 2018 +0200
Fix key sorting when using workspaces
---
buildstream/element.py | 31 ++++++++++++++++++++++---------
1 file changed, 22 insertions(+), 9 deletions(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index 331e329..37dfaaa 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -80,6 +80,7 @@ from collections import Mapping, OrderedDict
from contextlib import contextmanager
import tempfile
import shutil
+import functools
from . import _yaml
from ._variables import Variables
@@ -1137,21 +1138,33 @@ class Element(Plugin):
self.__build_result = None
return
+ def cmp_dep(a, b):
+ sysroot_a, key_a = a
+ sysroot_b, key_b = b
+ if key_a is None:
+ key_a = ''
+ if key_b is None:
+ key_b = ''
+ if (sysroot_a, key_a) < (sysroot_b, key_b):
+ return -1
+ elif (sysroot_a, key_a) > (sysroot_b, key_b):
+ return 1
+ else:
+ return 0
+
if self.__weak_cache_key is None:
# Calculate weak cache key
# Weak cache key includes names of direct build dependencies
# but does not include keys of dependencies.
dependencies = []
if self.BST_STRICT_REBUILD:
- for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True):
- dependencies = [(sysroot, e._get_cache_key(strength=_KeyStrength.WEAK))
- for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
+ dependencies = [(sysroot, e._get_cache_key(strength=_KeyStrength.WEAK))
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
else:
- for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True):
- dependencies = [(sysroot, e.name)
- for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
+ dependencies = [(sysroot, e.name)
+ for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
- self.__weak_cache_key = self.__calculate_cache_key(sorted(dependencies))
+ self.__weak_cache_key = self.__calculate_cache_key(sorted(dependencies, key=functools.cmp_to_key(cmp_dep)))
if self.__weak_cache_key is None:
# Weak cache key could not be calculated yet
@@ -1179,7 +1192,7 @@ class Element(Plugin):
if self.__strict_cache_key is None:
dependencies = [(sysroot, e.__strict_cache_key)
for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
- self.__strict_cache_key = self.__calculate_cache_key(sorted(dependencies))
+ self.__strict_cache_key = self.__calculate_cache_key(sorted(dependencies, key=functools.cmp_to_key(cmp_dep)))
if self.__strict_cache_key is None:
# Strict cache key could not be calculated yet
@@ -1220,7 +1233,7 @@ class Element(Plugin):
elif self.__assemble_scheduled or self.__assemble_done:
dependencies = [(sysroot, e._get_cache_key())
for sysroot, e in self.dependencies(Scope.BUILD, with_sysroot=True)]
- self.__cache_key = self.__calculate_cache_key(sorted(dependencies))
+ self.__cache_key = self.__calculate_cache_key(sorted(dependencies, key=functools.cmp_to_key(cmp_dep)))
if self.__cache_key is None:
# Strong cache key could not be calculated yet