You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:03:10 UTC
[buildstream] 01/03: element.py & loader: Support searching
elements using full paths
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch tristan/relative-search-paths
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 926a9186a226a5c2eb19d24f47eb2592c2edb484
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Fri Jul 24 23:37:51 2020 +0900
element.py & loader: Support searching elements using full paths
This makes the following changes:
* The LoadElements now keep a pointer to the resolved Elements, this
allows quick resolution of Elements via the regular loader search
resolution paths.
* The Loader now removes the _clear_caches() code, the caches of
loaded LoadElements are now persisted after the data model is loaded
so that quick Element.search() can be performed.
* The loader now exposes Loader.search_element() which returns a
LoadElement
* Element now implements Element.search() using the loader to resolve
a project relative element path to a LoadElement, and returns the
Element associated with the found LoadElement.
This fixes #931.
---
src/buildstream/_loader/loadelement.pyx | 15 +++++++++++++
src/buildstream/_loader/loader.py | 38 ++++++++++++++++-----------------
src/buildstream/_loader/metaelement.py | 3 +++
src/buildstream/element.py | 22 ++++++++++++-------
4 files changed, 51 insertions(+), 27 deletions(-)
diff --git a/src/buildstream/_loader/loadelement.pyx b/src/buildstream/_loader/loadelement.pyx
index de2f96b..2bf1260 100644
--- a/src/buildstream/_loader/loadelement.pyx
+++ b/src/buildstream/_loader/loadelement.pyx
@@ -84,6 +84,7 @@ cdef class LoadElement:
# TODO: if/when pyroaring exports symbols, we could type this statically
cdef object _dep_cache
cdef readonly list dependencies
+ cdef readonly object _element
def __cinit__(self, MappingNode node, str filename, object loader):
@@ -103,6 +104,7 @@ cdef class LoadElement:
#
self._loader = loader # The Loader object
self._dep_cache = None # The dependency cache, to speed up depends()
+ self._element = None # The instantiated Element, if any
#
# Initialization
@@ -150,6 +152,19 @@ cdef class LoadElement:
def junction(self):
return self._loader.project.junction
+ # element()
+ #
+ # The Element property stores the instantiated Element, if an
+ # Element has been instantiated for this LoadElement
+ #
+ @property
+ def element(self):
+ return self._element
+
+ @element.setter
+ def element(self, value):
+ self._element = value
+
# depends():
#
# Checks if this element depends on another element, directly
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index 9f88174..d2d14c3 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -170,8 +170,6 @@ class Loader:
#
ret.append(loader._collect_element(element))
- self._clean_caches()
-
# Cache how many Elements have just been loaded
if self.load_context.task:
# Workaround for task potentially being None (because no State object)
@@ -224,6 +222,24 @@ class Loader:
return loader
+ # search_element()
+ #
+ # Search the project for a given LoadElement
+ #
+ # Args:
+ # path (str): The relative element path
+ #
+ # Returns:
+ # (LoadElement): The LoadElement
+ #
+ # Raises:
+ # (LoadError): In case an element was not found, or would have
+ # required loading a project that is not yet loaded.
+ #
+ def search_element(self, path):
+ _, filename, loader = self._parse_name(path, None, load_subprojects=False)
+ return loader._load_file(filename, None, load_subprojects=False)
+
# ancestors()
#
# This will traverse all active loaders in the ancestry for which this
@@ -319,6 +335,7 @@ class Loader:
node.get_mapping(Symbol.PUBLIC, default={}),
node.get_mapping(Symbol.SANDBOX, default={}),
element_kind in ("junction", "link"),
+ element,
)
# Cache it now, make sure it's already there before recursing
@@ -905,20 +922,3 @@ class Loader:
),
warning_token=CoreWarnings.BAD_CHARACTERS_IN_NAME,
)
-
- # _clean_caches()
- #
- # Clean internal loader caches, recursively
- #
- # When loading the elements, the loaders use caches in order to not load the
- # same element twice. These are kept after loading and prevent garbage
- # collection. Cleaning them explicitely is required.
- #
- def _clean_caches(self):
- for loader in self._loaders.values():
- # value may be None with nested junctions without overrides
- if loader is not None:
- loader._clean_caches()
-
- self._meta_elements = {}
- self._elements = {}
diff --git a/src/buildstream/_loader/metaelement.py b/src/buildstream/_loader/metaelement.py
index 1c1f6fe..50d1bc5 100644
--- a/src/buildstream/_loader/metaelement.py
+++ b/src/buildstream/_loader/metaelement.py
@@ -39,6 +39,7 @@ class MetaElement:
# public: Public domain data dictionary
# sandbox: Configuration specific to the sandbox environment
# first_pass: The element is to be loaded with first pass configuration (junction)
+ # load_element (LoadElement): A reference back to the load_element
#
def __init__(
self,
@@ -54,6 +55,7 @@ class MetaElement:
public=None,
sandbox=None,
first_pass=False,
+ load_element=None,
):
self.project = project
self.name = name
@@ -71,3 +73,4 @@ class MetaElement:
self.strict_dependencies = []
self.first_pass = first_pass
self.is_junction = kind in ("junction", "link")
+ self.load_element = load_element
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 547397e..78f80bc 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -499,21 +499,23 @@ class Element(Plugin):
yield from visit(self, scope, visited)
- def search(self, scope: Scope, name: str) -> Optional["Element"]:
- """Search for a dependency by name
+ def search(self, scope: Scope, path: str) -> Optional["Element"]:
+ """Search for a dependency by it's element relative search path
+
+ The search path as a relative path to the element, including
+ any junction names leading up to the element which might
+ be in a subproject.
Args:
scope: The scope to search
- name: The dependency to search for
+ path: An element relative search path
Returns:
The dependency element, or None if not found.
"""
- for dep in self.dependencies(scope):
- if dep.name == name:
- return dep
-
- return None
+ project = self._get_project()
+ load_element = project.loader.search_element(path)
+ return load_element.element
def node_subst_vars(self, node: "ScalarNode") -> str:
"""Replace any variables in the string contained in the node and returns it.
@@ -908,6 +910,10 @@ class Element(Plugin):
element = meta.project.create_element(meta, first_pass=meta.first_pass)
cls.__instantiated_elements[meta] = element
+ # Store a reference on the originating LoadElement, allowing us
+ # to perform optimal Element.search().
+ meta.load_element.element = element
+
# Instantiate sources and generate their keys
for meta_source in meta.sources:
meta_source.first_pass = meta.is_junction