You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:13:48 UTC
[buildstream] 03/30: Make Project owner of Loader.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch valentindavid/flatpak-demo
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ea7bb36b997211f7d58dab8066130fd6e90499a4
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Mon Jun 11 15:57:28 2018 +0200
Make Project owner of Loader.
---
buildstream/_frontend/app.py | 30 +++++++++++++---------
buildstream/_loader/loader.py | 54 +++++++++++++++++++++------------------
buildstream/_pipeline.py | 11 +++-----
buildstream/_platform/linux.py | 4 +--
buildstream/_platform/platform.py | 3 +--
buildstream/_platform/unix.py | 4 +--
buildstream/_project.py | 8 +++++-
buildstream/_stream.py | 4 +--
tests/loader/__init__.py | 2 +-
9 files changed, 65 insertions(+), 55 deletions(-)
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index 4675b0e..217225d 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -34,6 +34,7 @@ from .. import Scope
# Import various buildstream internals
from .._context import Context
+from .._platform import Platform
from .._project import Project
from .._exceptions import BstError, StreamError, LoadError, LoadErrorReason, AppError
from .._message import Message, MessageType, unconditional_messages
@@ -66,6 +67,7 @@ class App():
self.context = None # The Context object
self.stream = None # The Stream object
self.project = None # The toplevel Project object
+ self.loader = None
self.logger = None # The LogLine object
self.interactive = None # Whether we are running in interactive mode
self.colors = None # Whether to use colors in logging
@@ -198,11 +200,26 @@ class App():
if option_value is not None:
setattr(self.context, context_attr, option_value)
+ Platform.create_instance(self.context)
+
+ # Create the logger right before setting the message handler
+ self.logger = LogLine(self.context,
+ self._content_profile,
+ self._format_profile,
+ self._success_profile,
+ self._error_profile,
+ self._detail_profile,
+ indent=INDENT)
+
+ # Propagate pipeline feedback to the user
+ self.context.set_message_handler(self._message_handler)
+
#
# Load the Project
#
try:
self.project = Project(directory, self.context, cli_options=self._main_options['option'])
+ self.loader = self.project.loader
except LoadError as e:
# Let's automatically start a `bst init` session in this case
@@ -217,24 +234,13 @@ class App():
except BstError as e:
self._error_exit(e, "Error loading project")
- # Create the logger right before setting the message handler
- self.logger = LogLine(self.context,
- self._content_profile,
- self._format_profile,
- self._success_profile,
- self._error_profile,
- self._detail_profile,
- indent=INDENT)
-
- # Propagate pipeline feedback to the user
- self.context.set_message_handler(self._message_handler)
-
# Now that we have a logger and message handler,
# we can override the global exception hook.
sys.excepthook = self._global_exception_handler
# Create the stream right away, we'll need to pass it around
self.stream = Stream(self.context, self.project, self._session_start,
+ self.loader,
session_start_callback=self.session_start_cb,
interrupt_callback=self._interrupt_handler,
ticker_callback=self._tick,
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index 11afef6..a190f17 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -25,7 +25,6 @@ import shutil
from .._exceptions import LoadError, LoadErrorReason
from .. import Consistency
-from .._project import Project
from .. import _yaml
from ..element import Element
from .._profile import Topics, profile_start, profile_end
@@ -49,11 +48,10 @@ from . import MetaSource
# parent (Loader): A parent Loader object, in the case this is a junctioned Loader
# tempdir (str): A directory to cleanup with the Loader, given to the loader by a parent
# loader in the case that this loader is a subproject loader.
-# fetch_subprojects (bool): Whether to fetch subprojects while loading
#
class Loader():
- def __init__(self, context, project, *, parent=None, tempdir=None, fetch_subprojects=False):
+ def __init__(self, context, project, *, parent=None, tempdir=None):
# Ensure we have an absolute path for the base directory
basedir = project.element_path
@@ -68,7 +66,6 @@ class Loader():
#
# Private members
#
- self._fetch_subprojects = fetch_subprojects
self._context = context
self._options = project.options # Project options (OptionPool)
self._basedir = basedir # Base project directory
@@ -88,11 +85,12 @@ class Loader():
# this is a bit more expensive due to deep copies
# ticker (callable): An optional function for tracking load progress
# targets (list of str): Target, element-path relative bst filenames in the project
+ # fetch_subprojects (bool): Whether to fetch subprojects while loading
#
# Raises: LoadError
#
# Returns: The toplevel LoadElement
- def load(self, targets, rewritable=False, ticker=None):
+ def load(self, targets, rewritable=False, ticker=None, fetch_subprojects=False):
for filename in targets:
if os.path.isabs(filename):
@@ -109,8 +107,9 @@ class Loader():
for target in targets:
profile_start(Topics.LOAD_PROJECT, target)
- junction, name, loader = self._parse_name(target, rewritable, ticker)
- loader._load_file(name, rewritable, ticker)
+ junction, name, loader = self._parse_name(target, rewritable, ticker,
+ fetch_subprojects=fetch_subprojects)
+ loader._load_file(name, rewritable, ticker, fetch_subprojects)
deps.append(Dependency(name, junction=junction))
profile_end(Topics.LOAD_PROJECT, target)
@@ -136,7 +135,8 @@ class Loader():
#
for target in targets:
profile_start(Topics.SORT_DEPENDENCIES, target)
- junction, name, loader = self._parse_name(target, rewritable, ticker)
+ junction, name, loader = self._parse_name(target, rewritable, ticker,
+ fetch_subprojects=fetch_subprojects)
loader._sort_dependencies(name)
profile_end(Topics.SORT_DEPENDENCIES, target)
# Finally, wrap what we have into LoadElements and return the target
@@ -197,11 +197,12 @@ class Loader():
# filename (str): The element-path relative bst file
# rewritable (bool): Whether we should load in round trippable mode
# ticker (callable): A callback to report loaded filenames to the frontend
+ # fetch_subprojects (bool): Whether to fetch subprojects while loading
#
# Returns:
# (LoadElement): A loaded LoadElement
#
- def _load_file(self, filename, rewritable, ticker):
+ def _load_file(self, filename, rewritable, ticker, fetch_subprojects):
# Silently ignore already loaded files
if filename in self._elements:
@@ -249,12 +250,13 @@ class Loader():
# Load all dependency files for the new LoadElement
for dep in element.deps:
if dep.junction:
- self._load_file(dep.junction, rewritable, ticker)
- loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker)
+ self._load_file(dep.junction, rewritable, ticker, fetch_subprojects)
+ loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
+ fetch_subprojects=fetch_subprojects)
else:
loader = self
- dep_element = loader._load_file(dep.name, rewritable, ticker)
+ dep_element = loader._load_file(dep.name, rewritable, ticker, fetch_subprojects)
if _yaml.node_get(dep_element.node, str, Symbol.KIND) == 'junction':
raise LoadError(LoadErrorReason.INVALID_DATA,
@@ -453,11 +455,12 @@ class Loader():
#
# Args:
# filename (str): Junction name
+ # fetch_subprojects (bool): Whether to fetch subprojects while loading
#
# Raises: LoadError
#
# Returns: A Loader or None if specified junction does not exist
- def _get_loader(self, filename, *, rewritable=False, ticker=None, level=0):
+ def _get_loader(self, filename, *, rewritable=False, ticker=None, level=0, fetch_subprojects=False):
# return previously determined result
if filename in self._loaders:
loader = self._loaders[filename]
@@ -474,13 +477,14 @@ class Loader():
if self._parent:
# junctions in the parent take precedence over junctions defined
# in subprojects
- loader = self._parent._get_loader(filename, rewritable=rewritable, ticker=ticker, level=level + 1)
+ loader = self._parent._get_loader(filename, rewritable=rewritable, ticker=ticker,
+ level=level + 1, fetch_subprojects=fetch_subprojects)
if loader:
self._loaders[filename] = loader
return loader
try:
- self._load_file(filename, rewritable, ticker)
+ self._load_file(filename, rewritable, ticker, fetch_subprojects)
except LoadError as e:
if e.reason != LoadErrorReason.MISSING_FILE:
# other load error
@@ -509,7 +513,7 @@ class Loader():
# Handle the case where a subproject needs to be fetched
#
if source.get_consistency() == Consistency.RESOLVED:
- if self._fetch_subprojects:
+ if fetch_subprojects:
if ticker:
ticker(filename, 'Fetching subproject from {} source'.format(source.get_kind()))
source.fetch()
@@ -535,7 +539,9 @@ class Loader():
# Load the project
project_dir = os.path.join(basedir, element.path)
try:
- project = Project(project_dir, self._context, junction=element)
+ from .._project import Project
+ project = Project(project_dir, self._context, junction=element,
+ parent_loader=self, tempdir=basedir)
except LoadError as e:
if e.reason == LoadErrorReason.MISSING_PROJECT_CONF:
raise LoadError(reason=LoadErrorReason.INVALID_JUNCTION,
@@ -545,11 +551,7 @@ class Loader():
else:
raise
- loader = Loader(self._context, project,
- parent=self,
- tempdir=basedir,
- fetch_subprojects=self._fetch_subprojects)
-
+ loader = project.loader
self._loaders[filename] = loader
return loader
@@ -580,13 +582,14 @@ class Loader():
# rewritable (bool): Whether the loaded files should be rewritable
# this is a bit more expensive due to deep copies
# ticker (callable): An optional function for tracking load progress
+ # fetch_subprojects (bool): Whether to fetch subprojects while loading
#
# Returns:
# (tuple): - (str): name of the junction element
# - (str): name of the element
# - (Loader): loader for sub-project
#
- def _parse_name(self, name, rewritable, ticker):
+ def _parse_name(self, name, rewritable, ticker, fetch_subprojects=False):
# We allow to split only once since deep junctions names are forbidden.
# Users who want to refer to elements in sub-sub-projects are required
# to create junctions on the top level project.
@@ -594,6 +597,7 @@ class Loader():
if len(junction_path) == 1:
return None, junction_path[-1], self
else:
- self._load_file(junction_path[-2], rewritable, ticker)
- loader = self._get_loader(junction_path[-2], rewritable=rewritable, ticker=ticker)
+ self._load_file(junction_path[-2], rewritable, ticker, fetch_subprojects)
+ loader = self._get_loader(junction_path[-2], rewritable=rewritable, ticker=ticker,
+ fetch_subprojects=fetch_subprojects)
return junction_path[-2], junction_path[-1], loader
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index 1474b37..909ae24 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -25,7 +25,6 @@ from operator import itemgetter
from ._exceptions import PipelineError
from ._message import Message, MessageType
-from ._loader import Loader
from ._profile import Topics, profile_start, profile_end
from .element import Element
from . import Scope, Consistency
@@ -77,7 +76,6 @@ class Pipeline():
# Private members
#
self._artifacts = artifacts
- self._loader = None
# load()
#
@@ -106,11 +104,9 @@ class Pipeline():
profile_start(Topics.LOAD_PIPELINE, "_".join(t.replace(os.sep, '-') for t in targets))
- self._loader = Loader(self._context, self._project,
- fetch_subprojects=fetch_subprojects)
-
with self._context.timed_activity("Loading pipeline", silent_nested=True):
- meta_elements = self._loader.load(targets, rewritable, None)
+ meta_elements = self._project.loader.load(targets, rewritable, None,
+ fetch_subprojects=fetch_subprojects)
# Resolve the real elements now that we've loaded the project
with self._context.timed_activity("Resolving pipeline"):
@@ -388,8 +384,7 @@ class Pipeline():
# Cleans up resources used by the Pipeline.
#
def cleanup(self):
- if self._loader:
- self._loader.cleanup()
+ self._project.loader.cleanup()
# Reset the element loader state
Element._reset_load_state()
diff --git a/buildstream/_platform/linux.py b/buildstream/_platform/linux.py
index fec512b..e6541e8 100644
--- a/buildstream/_platform/linux.py
+++ b/buildstream/_platform/linux.py
@@ -30,9 +30,9 @@ from . import Platform
class Linux(Platform):
- def __init__(self, context, project):
+ def __init__(self, context):
- super().__init__(context, project)
+ super().__init__(context)
self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
self._user_ns_available = self._check_user_ns_available(context)
diff --git a/buildstream/_platform/platform.py b/buildstream/_platform/platform.py
index 29da335..8a074eb 100644
--- a/buildstream/_platform/platform.py
+++ b/buildstream/_platform/platform.py
@@ -35,9 +35,8 @@ class Platform():
# Args:
# context (context): The project context
#
- def __init__(self, context, project):
+ def __init__(self, context):
self.context = context
- self.project = project
@classmethod
def create_instance(cls, *args, **kwargs):
diff --git a/buildstream/_platform/unix.py b/buildstream/_platform/unix.py
index 8b1d2ec..edbd355 100644
--- a/buildstream/_platform/unix.py
+++ b/buildstream/_platform/unix.py
@@ -28,9 +28,9 @@ from . import Platform
class Unix(Platform):
- def __init__(self, context, project):
+ def __init__(self, context):
- super().__init__(context, project)
+ super().__init__(context)
self._artifact_cache = TarCache(context)
# Not necessarily 100% reliable, but we want to fail early.
diff --git a/buildstream/_project.py b/buildstream/_project.py
index b568cf8..36ae5b2 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -33,6 +33,7 @@ from ._elementfactory import ElementFactory
from ._sourcefactory import SourceFactory
from ._projectrefs import ProjectRefs, ProjectRefStorage
from ._versions import BST_FORMAT_VERSION
+from ._loader import Loader
# The separator we use for user specified aliases
@@ -70,7 +71,8 @@ class HostMount():
#
class Project():
- def __init__(self, directory, context, *, junction=None, cli_options=None):
+ def __init__(self, directory, context, *, junction=None, cli_options=None,
+ parent_loader=None, tempdir=None):
# The project name
self.name = None
@@ -118,6 +120,10 @@ class Project():
self._context.add_project(self)
+ self.loader = Loader(self._context, self,
+ parent=parent_loader,
+ tempdir=tempdir)
+
# translate_url():
#
# Translates the given url which may be specified with an alias
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index 5013daf..48d3571 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -44,6 +44,7 @@ from . import Scope, Consistency
# context (Context): The Context object
# project (Project): The Project object
# session_start (datetime): The time when the session started
+# loader (Loader): The Loader object
# session_start_callback (callable): A callback to invoke when the session starts
# interrupt_callback (callable): A callback to invoke when we get interrupted
# ticker_callback (callable): Invoked every second while running the scheduler
@@ -52,7 +53,7 @@ from . import Scope, Consistency
#
class Stream():
- def __init__(self, context, project, session_start, *,
+ def __init__(self, context, project, session_start, loader, *,
session_start_callback=None,
interrupt_callback=None,
ticker_callback=None,
@@ -70,7 +71,6 @@ class Stream():
#
# Private members
#
- Platform.create_instance(context, project)
self._platform = Platform.get_platform()
self._artifacts = self._platform.artifactcache
self._context = context
diff --git a/tests/loader/__init__.py b/tests/loader/__init__.py
index 49db9cf..fcefdac 100644
--- a/tests/loader/__init__.py
+++ b/tests/loader/__init__.py
@@ -11,4 +11,4 @@ from buildstream._loader import Loader
def make_loader(basedir):
context = Context()
project = Project(basedir, context)
- return Loader(context, project)
+ return project.loader