You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:13:50 UTC
[buildstream] 05/30: Add support for include '(@)' in project.conf
and .bst files
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch valentindavid/flatpak-demo
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 292766d8f3d36f60d152762ebe928a93ef64d40e
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Wed Jun 13 13:15:15 2018 +0200
Add support for include '(@)' in project.conf and .bst files
Fixes #331.
---
buildstream/_includes.py | 48 +++++
buildstream/_loader/loader.py | 26 ++-
buildstream/_loader/metasource.py | 1 +
buildstream/_options/optionpool.py | 12 +-
buildstream/_project.py | 186 ++++++++++++-------
buildstream/_stream.py | 1 +
buildstream/element.py | 78 ++++++--
buildstream/plugins/elements/junction.py | 1 +
buildstream/source.py | 10 +-
tests/format/include.py | 200 +++++++++++++++++++++
tests/format/include/defines_name/element.bst | 1 +
tests/format/include/defines_name/extra_conf.yml | 1 +
tests/format/include/defines_name/project.conf | 4 +
tests/format/include/file/element.bst | 1 +
tests/format/include/file/extra_conf.yml | 2 +
tests/format/include/file/project.conf | 4 +
.../include/file_with_subproject/element.bst | 1 +
.../include/file_with_subproject/extra_conf.yml | 2 +
.../include/file_with_subproject/project.bst | 4 +
.../include/file_with_subproject/project.conf | 4 +
.../file_with_subproject/subproject/project.conf | 1 +
tests/format/include/junction/element.bst | 1 +
tests/format/include/junction/project.conf | 4 +
.../include/junction/subproject/extra_conf.yml | 2 +
.../include/junction/subproject/project.conf | 1 +
tests/format/include/options/element.bst | 1 +
tests/format/include/options/extra_conf.yml | 8 +
tests/format/include/options/project.conf | 4 +
tests/format/include/overrides/element.bst | 1 +
tests/format/include/overrides/extra_conf.yml | 16 ++
tests/format/include/overrides/project.conf | 20 +++
.../include/overrides/subproject/project.conf | 1 +
tests/format/include/sub-include/element.bst | 1 +
tests/format/include/sub-include/manual_conf.yml | 2 +
tests/format/include/sub-include/project.conf | 6 +
35 files changed, 562 insertions(+), 94 deletions(-)
diff --git a/buildstream/_includes.py b/buildstream/_includes.py
new file mode 100644
index 0000000..718cd82
--- /dev/null
+++ b/buildstream/_includes.py
@@ -0,0 +1,48 @@
+import os
+from collections import Mapping
+from . import _yaml
+
+
+class Includes:
+
+ def __init__(self, loader, valid_keys=None):
+ self._loader = loader
+ self._valid_keys = valid_keys
+
+ def process(self, node):
+ while True:
+ includes = _yaml.node_get(node, list, '(@)', default_value=None)
+ if '(@)' in node:
+ del node['(@)']
+
+ if not includes:
+ break
+
+ for include in includes:
+ include_node = self._include_file(include)
+ if self._valid_keys:
+ _yaml.node_validate(include_node, self._valid_keys)
+
+ _yaml.composite(node, include_node)
+
+ for _, value in _yaml.node_items(node):
+ self._process_value(value)
+
+ def _include_file(self, include):
+ if ':' in include:
+ junction, include = include.split(':', 1)
+ junction_loader = self._loader._get_loader(junction, fetch_subprojects=True)
+ directory = junction_loader.project.directory
+ else:
+ directory = self._loader.project.directory
+ return _yaml.load(os.path.join(directory, include))
+
+ def _process_value(self, value):
+ if isinstance(value, Mapping):
+ self.process(value)
+ elif isinstance(value, list):
+ self._process_list(value)
+
+ def _process_list(self, values):
+ for value in values:
+ self._process_value(value)
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index a190f17..c5bce62 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -23,12 +23,13 @@ from collections import Mapping, namedtuple
import tempfile
import shutil
-from .._exceptions import LoadError, LoadErrorReason
+from .._exceptions import LoadError, LoadErrorReason, PluginError
from .. import Consistency
from .. import _yaml
from ..element import Element
from .._profile import Topics, profile_start, profile_end
from .._platform import Platform
+from .._includes import Includes
from .types import Symbol, Dependency
from .loadelement import LoadElement
@@ -69,6 +70,7 @@ class Loader():
self._context = context
self._options = project.options # Project options (OptionPool)
self._basedir = basedir # Base project directory
+ self._first_pass_options = project.first_pass_config.options # Project options (OptionPool)
self._tempdir = tempdir # A directory to cleanup
self._parent = parent # The parent loader
@@ -76,6 +78,8 @@ class Loader():
self._elements = {} # Dict of elements
self._loaders = {} # Dict of junction loaders
+ self._includes = Includes(self)
+
# load():
#
# Loads the project based on the parameters given to the constructor
@@ -241,7 +245,22 @@ class Loader():
message, detail=detail) from e
else:
raise
- self._options.process_node(node)
+ kind = _yaml.node_get(node, str, Symbol.KIND)
+ try:
+ kind_type, _ = self.project.first_pass_config.plugins.get_element_type(kind)
+ except PluginError:
+ kind_type = None
+ if kind_type and hasattr(kind_type, 'BST_NO_PROJECT_DEFAULTS') and kind_type.BST_NO_PROJECT_DEFAULTS:
+ self._first_pass_options.process_node(node)
+ else:
+ if not self.project.is_loaded():
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Cannot pre-load. Element depends on project defaults."
+ .format(filename))
+
+ self._includes.process(node)
+
+ self._options.process_node(node)
element = LoadElement(node, filename, self)
@@ -506,7 +525,8 @@ class Loader():
"{}: Expected junction but element kind is {}".format(filename, meta_element.kind))
platform = Platform.get_platform()
- element = Element._new_from_meta(meta_element, platform.artifactcache)
+ element = Element._new_from_meta(meta_element, platform.artifactcache,
+ first_pass=True)
element._preflight()
for source in element.sources():
diff --git a/buildstream/_loader/metasource.py b/buildstream/_loader/metasource.py
index 3bcc21e..4241ae5 100644
--- a/buildstream/_loader/metasource.py
+++ b/buildstream/_loader/metasource.py
@@ -38,3 +38,4 @@ class MetaSource():
self.kind = kind
self.config = config
self.directory = directory
+ self.first_pass = False
diff --git a/buildstream/_options/optionpool.py b/buildstream/_options/optionpool.py
index f90fd82..83a202f 100644
--- a/buildstream/_options/optionpool.py
+++ b/buildstream/_options/optionpool.py
@@ -108,15 +108,17 @@ class OptionPool():
# Args:
# cli_options (list): A list of (str, str) tuples
#
- def load_cli_values(self, cli_options):
+ def load_cli_values(self, cli_options, ignore_unknown=False):
for option_name, option_value in cli_options:
try:
option = self._options[option_name]
except KeyError as e:
- raise LoadError(LoadErrorReason.INVALID_DATA,
- "Unknown option '{}' specified on the command line"
- .format(option_name)) from e
- option.set_value(option_value)
+ if not ignore_unknown:
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "Unknown option '{}' specified on the command line"
+ .format(option_name)) from e
+ else:
+ option.set_value(option_value)
# resolve()
#
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 0668adc..b4aa1ef 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -34,6 +34,7 @@ from ._sourcefactory import SourceFactory
from ._projectrefs import ProjectRefs, ProjectRefStorage
from ._versions import BST_FORMAT_VERSION
from ._loader import Loader
+from ._includes import Includes
# The separator we use for user specified aliases
@@ -199,12 +200,33 @@ class PluginCollection:
.format(plugin, plugin.BST_FORMAT_VERSION, version))
+class ProjectConfig:
+ def __init__(self):
+ self.plugins = None
+ self.options = None # OptionPool
+ self.base_variables = {} # The base set of variables
+ self.element_overrides = {} # Element specific configurations
+ self.source_overrides = {} # Source specific configurations
+
+
# Project()
#
# The Project Configuration
#
class Project():
+ INCLUDE_CONFIG_KEYS = ['variables',
+ 'environment', 'environment-nocache',
+ 'split-rules', 'elements', 'plugins',
+ 'aliases', 'artifacts',
+ 'fail-on-overlap', 'shell',
+ 'ref-storage', 'sandbox',
+ 'options']
+
+ MAIN_FILE_CONFIG_KEYS = ['format-version',
+ 'element-path',
+ 'name']
+
def __init__(self, directory, context, *, junction=None, cli_options=None,
parent_loader=None, tempdir=None):
@@ -221,16 +243,14 @@ class Project():
self.refs = ProjectRefs(self.directory, 'project.refs')
self.junction_refs = ProjectRefs(self.directory, 'junction.refs')
- self.plugins = None # PluginCollection
- self.options = None # OptionPool
+ self.config = ProjectConfig()
+ self.first_pass_config = ProjectConfig()
+
self.junction = junction # The junction Element object, if this is a subproject
self.fail_on_overlap = False # Whether overlaps are treated as errors
self.ref_storage = None # ProjectRefStorage setting
- self.base_variables = {} # The base set of variables
self.base_environment = {} # The base set of environment variables
self.base_env_nocache = None # The base nocache mask (list) for the environment
- self.element_overrides = {} # Element specific configurations
- self.source_overrides = {} # Source specific configurations
#
# Private Members
@@ -245,15 +265,42 @@ class Project():
self._shell_environment = {} # Statically set environment vars
self._shell_host_files = [] # A list of HostMount objects
+ self.artifact_cache_specs = None
+ self._sandbox = None
+ self._splits = None
+
+ self._context.add_project(self)
+
+ self._loaded = False
+
profile_start(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
- self._load()
+ self._load(parent_loader=parent_loader, tempdir=tempdir)
profile_end(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
- self._context.add_project(self)
+ self._loaded = True
- self.loader = Loader(self._context, self,
- parent=parent_loader,
- tempdir=tempdir)
+ @property
+ def plugins(self):
+ return self.config.plugins
+
+ @property
+ def options(self):
+ return self.config.options
+
+ @property
+ def base_variables(self):
+ return self.config.base_variables
+
+ @property
+ def element_overrides(self):
+ return self.config.element_overrides
+
+ @property
+ def source_overrides(self):
+ return self.config.source_overrides
+
+ def is_loaded(self):
+ return self._loaded
# translate_url():
#
@@ -312,7 +359,7 @@ class Project():
#
# Raises: LoadError if there was a problem with the project.conf
#
- def _load(self):
+ def _load(self, parent_loader=None, tempdir=None):
# Load builtin default
projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE)
@@ -327,15 +374,6 @@ class Project():
_yaml.composite(config, project_conf)
- # Element and Source type configurations will be composited later onto
- # element/source types, so we delete it from here and run our final
- # assertion after.
- self.element_overrides = _yaml.node_get(config, Mapping, 'elements', default_value={})
- self.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
- config.pop('elements', None)
- config.pop('sources', None)
- _yaml.node_final_assertions(config)
-
# Assert project's format version early, before validating toplevel keys
format_version = _yaml.node_get(config, int, 'format-version')
if BST_FORMAT_VERSION < format_version:
@@ -345,17 +383,6 @@ class Project():
"Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
.format(format_version, major, minor, BST_FORMAT_VERSION))
- _yaml.node_validate(config, [
- 'format-version',
- 'element-path', 'variables',
- 'environment', 'environment-nocache',
- 'split-rules', 'elements', 'plugins',
- 'aliases', 'name',
- 'artifacts', 'options',
- 'fail-on-overlap', 'shell',
- 'ref-storage', 'sandbox'
- ])
-
# The project name, element path and option declarations
# are constant and cannot be overridden by option conditional statements
self.name = _yaml.node_get(config, str, 'name')
@@ -369,30 +396,21 @@ class Project():
_yaml.node_get(config, str, 'element-path')
)
- # Load project options
- options_node = _yaml.node_get(config, Mapping, 'options', default_value={})
- self.options = OptionPool(self.element_path)
- self.options.load(options_node)
- if self.junction:
- # load before user configuration
- self.options.load_yaml_values(self.junction.options, transform=self.junction._subst_string)
+ self.config.options = OptionPool(self.element_path)
+ self.first_pass_config.options = OptionPool(self.element_path)
- # Collect option values specified in the user configuration
- overrides = self._context.get_overrides(self.name)
- override_options = _yaml.node_get(overrides, Mapping, 'options', default_value={})
- self.options.load_yaml_values(override_options)
- if self._cli_options:
- self.options.load_cli_values(self._cli_options)
+ self.loader = Loader(self._context, self,
+ parent=parent_loader,
+ tempdir=tempdir)
- # We're done modifying options, now we can use them for substitutions
- self.options.resolve()
+ self._load_pass(_yaml.node_copy(config), self.first_pass_config, True)
- #
- # Now resolve any conditionals in the remaining configuration,
- # any conditionals specified for project option declarations,
- # or conditionally specifying the project name; will be ignored.
- #
- self.options.process_node(config)
+ project_includes = Includes(self.loader, self.INCLUDE_CONFIG_KEYS + ['elements', 'sources'])
+ project_includes.process(config)
+
+ self._load_pass(config, self.config, False)
+
+ _yaml.node_validate(config, self.INCLUDE_CONFIG_KEYS + self.MAIN_FILE_CONFIG_KEYS)
#
# Now all YAML composition is done, from here on we just load
@@ -402,23 +420,9 @@ class Project():
# Load artifacts pull/push configuration for this project
self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config)
- self.plugins = PluginCollection(self, self._context, self.directory, config)
# Source url aliases
self._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
- # Load base variables
- self.base_variables = _yaml.node_get(config, Mapping, 'variables')
-
- # Add the project name as a default variable
- self.base_variables['project-name'] = self.name
-
- # Extend variables with automatic variables and option exports
- # Initialize it as a string as all variables are processed as strings.
- self.base_variables['max-jobs'] = str(multiprocessing.cpu_count())
-
- # Export options into variables, if that was requested
- self.options.export_variables(self.base_variables)
-
# Load sandbox environment variables
self.base_environment = _yaml.node_get(config, Mapping, 'environment')
self.base_env_nocache = _yaml.node_get(config, list, 'environment-nocache')
@@ -475,6 +479,56 @@ class Project():
self._shell_host_files.append(mount)
+ def _load_pass(self, config, output, ignore_unknown):
+
+ # Element and Source type configurations will be composited later onto
+ # element/source types, so we delete it from here and run our final
+ # assertion after.
+ output.element_overrides = _yaml.node_get(config, Mapping, 'elements', default_value={})
+ output.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
+ config.pop('elements', None)
+ config.pop('sources', None)
+ _yaml.node_final_assertions(config)
+
+ output.plugins = PluginCollection(self, self._context, self.directory, config)
+
+ # Load project options
+ options_node = _yaml.node_get(config, Mapping, 'options', default_value={})
+ output.options.load(options_node)
+ if self.junction:
+ # load before user configuration
+ output.options.load_yaml_values(self.junction.options, transform=self.junction._subst_string)
+
+ # Collect option values specified in the user configuration
+ overrides = self._context.get_overrides(self.name)
+ override_options = _yaml.node_get(overrides, Mapping, 'options', default_value={})
+ output.options.load_yaml_values(override_options)
+ if self._cli_options:
+ output.options.load_cli_values(self._cli_options, ignore_unknown=ignore_unknown)
+
+ # We're done modifying options, now we can use them for substitutions
+ output.options.resolve()
+
+ #
+ # Now resolve any conditionals in the remaining configuration,
+ # any conditionals specified for project option declarations,
+ # or conditionally specifying the project name; will be ignored.
+ #
+ output.options.process_node(config)
+
+ # Load base variables
+ output.base_variables = _yaml.node_get(config, Mapping, 'variables')
+
+ # Add the project name as a default variable
+ output.base_variables['project-name'] = self.name
+
+ # Extend variables with automatic variables and option exports
+ # Initialize it as a string as all variables are processed as strings.
+ output.base_variables['max-jobs'] = str(multiprocessing.cpu_count())
+
+ # Export options into variables, if that was requested
+ output.options.export_variables(output.base_variables)
+
# _ensure_project_dir()
#
# Returns path of the project directory, if a configuration file is found
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index 48d3571..4801ecc 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -75,6 +75,7 @@ class Stream():
self._artifacts = self._platform.artifactcache
self._context = context
self._project = project
+
self._pipeline = Pipeline(context, project, self._artifacts)
self._scheduler = Scheduler(context, session_start,
interrupt_callback=interrupt_callback,
diff --git a/buildstream/element.py b/buildstream/element.py
index f8a993a..ee523ff 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -191,10 +191,19 @@ class Element(Plugin):
*Since: 1.2*
"""
+ BST_NO_PROJECT_DEFAULTS = False
+ """
+
+ """
+
def __init__(self, context, project, artifacts, meta, plugin_conf):
super().__init__(meta.name, context, project, meta.provenance, "element")
+ if not project.is_loaded() and not self.BST_NO_PROJECT_DEFAULTS:
+ raise ElementError("{}: Cannot load element before project"
+ .format(self), reason="project-not-loaded")
+
self.normal_name = os.path.splitext(self.name.replace(os.sep, '-'))[0]
"""A normalized element name
@@ -885,9 +894,12 @@ class Element(Plugin):
# (Element): A newly created Element instance
#
@classmethod
- def _new_from_meta(cls, meta, artifacts):
+ def _new_from_meta(cls, meta, artifacts, first_pass=False):
- plugins = meta.project.plugins
+ if first_pass:
+ plugins = meta.project.first_pass_config.plugins
+ else:
+ plugins = meta.project.plugins
if meta in cls.__instantiated_elements:
return cls.__instantiated_elements[meta]
@@ -897,6 +909,7 @@ class Element(Plugin):
# Instantiate sources
for meta_source in meta.sources:
+ meta_source.first_pass = element.BST_NO_PROJECT_DEFAULTS
source = plugins.create_source(meta_source)
redundant_ref = source._load_ref()
element.__sources.append(source)
@@ -907,10 +920,10 @@ class Element(Plugin):
# Instantiate dependencies
for meta_dep in meta.dependencies:
- dependency = Element._new_from_meta(meta_dep, artifacts)
+ dependency = Element._new_from_meta(meta_dep, artifacts, first_pass=first_pass)
element.__runtime_dependencies.append(dependency)
for meta_dep in meta.build_dependencies:
- dependency = Element._new_from_meta(meta_dep, artifacts)
+ dependency = Element._new_from_meta(meta_dep, artifacts, first_pass=first_pass)
element.__build_dependencies.append(dependency)
return element
@@ -2095,16 +2108,24 @@ class Element(Plugin):
def __compose_default_splits(self, defaults):
project = self._get_project()
- project_splits = _yaml.node_chain_copy(project._splits)
element_public = _yaml.node_get(defaults, Mapping, 'public', default_value={})
element_bst = _yaml.node_get(element_public, Mapping, 'bst', default_value={})
element_splits = _yaml.node_get(element_bst, Mapping, 'split-rules', default_value={})
- # Extend project wide split rules with any split rules defined by the element
- _yaml.composite(project_splits, element_splits)
+ if self.BST_NO_PROJECT_DEFAULTS:
+ splits = _yaml.node_chain_copy(element_splits)
+ elif project._splits is None:
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Project was not fully loaded while loading element. "
+ "Only non-artifact elements (e.g. junctions) are allowed in this context."
+ .format(self.name))
+ else:
+ splits = _yaml.node_chain_copy(project._splits)
+ # Extend project wide split rules with any split rules defined by the element
+ _yaml.composite(splits, element_splits)
- element_bst['split-rules'] = project_splits
+ element_bst['split-rules'] = splits
element_public['bst'] = element_bst
defaults['public'] = element_public
@@ -2128,7 +2149,11 @@ class Element(Plugin):
# Override the element's defaults with element specific
# overrides from the project.conf
project = self._get_project()
- elements = project.element_overrides
+ if self.BST_NO_PROJECT_DEFAULTS:
+ elements = project.first_pass_config.element_overrides
+ else:
+ elements = project.element_overrides
+
overrides = elements.get(self.get_kind())
if overrides:
_yaml.composite(defaults, overrides)
@@ -2141,10 +2166,14 @@ class Element(Plugin):
# creating sandboxes for this element
#
def __extract_environment(self, meta):
- project = self._get_project()
default_env = _yaml.node_get(self.__defaults, Mapping, 'environment', default_value={})
- environment = _yaml.node_chain_copy(project.base_environment)
+ if self.BST_NO_PROJECT_DEFAULTS:
+ environment = {}
+ else:
+ project = self._get_project()
+ environment = _yaml.node_chain_copy(project.base_environment)
+
_yaml.composite(environment, default_env)
_yaml.composite(environment, meta.environment)
_yaml.node_final_assertions(environment)
@@ -2157,8 +2186,13 @@ class Element(Plugin):
return final_env
def __extract_env_nocache(self, meta):
- project = self._get_project()
- project_nocache = project.base_env_nocache
+ if self.BST_NO_PROJECT_DEFAULTS:
+ project_nocache = []
+ else:
+ project = self._get_project()
+ assert project.is_loaded()
+ project_nocache = project.base_env_nocache
+
default_nocache = _yaml.node_get(self.__defaults, list, 'environment-nocache', default_value=[])
element_nocache = meta.env_nocache
@@ -2173,10 +2207,15 @@ class Element(Plugin):
# substituting command strings to be run in the sandbox
#
def __extract_variables(self, meta):
- project = self._get_project()
default_vars = _yaml.node_get(self.__defaults, Mapping, 'variables', default_value={})
- variables = _yaml.node_chain_copy(project.base_variables)
+ project = self._get_project()
+ if self.BST_NO_PROJECT_DEFAULTS:
+ variables = _yaml.node_chain_copy(project.first_pass_config.base_variables)
+ else:
+ assert project.is_loaded()
+ variables = _yaml.node_chain_copy(project.base_variables)
+
_yaml.composite(variables, default_vars)
_yaml.composite(variables, meta.variables)
_yaml.node_final_assertions(variables)
@@ -2200,13 +2239,18 @@ class Element(Plugin):
# Sandbox-specific configuration data, to be passed to the sandbox's constructor.
#
def __extract_sandbox_config(self, meta):
- project = self._get_project()
+ if self.BST_NO_PROJECT_DEFAULTS:
+ sandbox_config = {'build-uid': 0,
+ 'build-gid': 0}
+ else:
+ project = self._get_project()
+ assert project.is_loaded()
+ sandbox_config = _yaml.node_chain_copy(project._sandbox)
# The default config is already composited with the project overrides
sandbox_defaults = _yaml.node_get(self.__defaults, Mapping, 'sandbox', default_value={})
sandbox_defaults = _yaml.node_chain_copy(sandbox_defaults)
- sandbox_config = _yaml.node_chain_copy(project._sandbox)
_yaml.composite(sandbox_config, sandbox_defaults)
_yaml.composite(sandbox_config, meta.sandbox)
_yaml.node_final_assertions(sandbox_config)
diff --git a/buildstream/plugins/elements/junction.py b/buildstream/plugins/elements/junction.py
index ee5ed24..2f81f46 100644
--- a/buildstream/plugins/elements/junction.py
+++ b/buildstream/plugins/elements/junction.py
@@ -136,6 +136,7 @@ class JunctionElement(Element):
# Junctions are not allowed any dependencies
BST_FORBID_BDEPENDS = True
BST_FORBID_RDEPENDS = True
+ BST_NO_PROJECT_DEFAULTS = True
def configure(self, node):
self.path = self.node_get_member(node, str, 'path', default='')
diff --git a/buildstream/source.py b/buildstream/source.py
index ec38ae8..c019934 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -137,8 +137,9 @@ class Source(Plugin):
# Collect the composited element configuration and
# ask the element to configure itself.
- self.__init_defaults()
+ self.__init_defaults(meta)
self.__config = self.__extract_config(meta)
+
self.configure(self.__config)
COMMON_CONFIG_KEYS = ['kind', 'directory']
@@ -611,10 +612,13 @@ class Source(Plugin):
reason="ensure-stage-dir-fail") from e
return directory
- def __init_defaults(self):
+ def __init_defaults(self, meta):
if not self.__defaults_set:
project = self._get_project()
- sources = project.source_overrides
+ if meta.first_pass:
+ sources = project.first_pass_config.source_overrides
+ else:
+ sources = project.source_overrides
type(self).__defaults = sources.get(self.get_kind(), {})
type(self).__defaults_set = True
diff --git a/tests/format/include.py b/tests/format/include.py
new file mode 100644
index 0000000..ca6eaab
--- /dev/null
+++ b/tests/format/include.py
@@ -0,0 +1,200 @@
+import os
+import pytest
+from buildstream import _yaml
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
+from tests.testutils import cli, generate_junction, create_repo
+
+
+# Project directory
+DATA_DIR = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)),
+ 'include'
+)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_project_file(cli, datafiles):
+ project = os.path.join(str(datafiles), 'file')
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert loaded['included'] == 'True'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_junction_file(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'junction')
+
+ generate_junction(tmpdir,
+ os.path.join(project, 'subproject'),
+ os.path.join(project, 'junction.bst'),
+ store_ref=True)
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert loaded['included'] == 'True'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_junction_options(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'options')
+
+ result = cli.run(project=project, args=[
+ '-o', 'build_arch', 'x86_64',
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert loaded['build_arch'] == 'x86_64'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_project_defines_name(cli, datafiles):
+ project = os.path.join(str(datafiles), 'defines_name')
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_junction_element_partial_project_project(cli, tmpdir, datafiles):
+ """
+ Junction elements never depend on fully include processed project.
+ """
+
+ project = os.path.join(str(datafiles), 'junction')
+
+ subproject_path = os.path.join(project, 'subproject')
+ junction_path = os.path.join(project, 'junction.bst')
+
+ repo = create_repo('git', str(tmpdir))
+
+ ref = repo.create(subproject_path)
+
+ element = {
+ 'kind': 'junction',
+ 'sources': [
+ repo.source_config(ref=ref)
+ ]
+ }
+ _yaml.dump(element, junction_path)
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'junction.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'included' not in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_junction_element_partial_project_file(cli, tmpdir, datafiles):
+ """
+ Junction elements never depend on fully include processed project.
+ """
+
+ project = os.path.join(str(datafiles), 'file_with_subproject')
+
+ subproject_path = os.path.join(project, 'subproject')
+ junction_path = os.path.join(project, 'junction.bst')
+
+ repo = create_repo('git', str(tmpdir))
+
+ ref = repo.create(subproject_path)
+
+ element = {
+ 'kind': 'junction',
+ 'sources': [
+ repo.source_config(ref=ref)
+ ]
+ }
+ _yaml.dump(element, junction_path)
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'junction.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'included' not in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_element_overrides(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'overrides')
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'manual_main_override' in loaded
+ assert 'manual_included_override' in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_element_overrides_composition(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'overrides')
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{config}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'build-commands' in loaded
+ assert loaded['build-commands'] == ['first', 'second']
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_element_overrides_sub_include(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'sub-include')
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'element.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'included' in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_junction_do_not_use_included_overrides(cli, tmpdir, datafiles):
+ project = os.path.join(str(datafiles), 'overrides')
+
+ generate_junction(tmpdir,
+ os.path.join(project, 'subproject'),
+ os.path.join(project, 'junction.bst'),
+ store_ref=True)
+
+ result = cli.run(project=project, args=[
+ 'show',
+ '--deps', 'none',
+ '--format', '%{vars}',
+ 'junction.bst'])
+ result.assert_success()
+ loaded = _yaml.load_data(result.output)
+ assert 'main_override' in loaded
+ assert 'included_override' not in loaded
diff --git a/tests/format/include/defines_name/element.bst b/tests/format/include/defines_name/element.bst
new file mode 100644
index 0000000..4d7f702
--- /dev/null
+++ b/tests/format/include/defines_name/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/defines_name/extra_conf.yml b/tests/format/include/defines_name/extra_conf.yml
new file mode 100644
index 0000000..84e8c6a
--- /dev/null
+++ b/tests/format/include/defines_name/extra_conf.yml
@@ -0,0 +1 @@
+name: othername
diff --git a/tests/format/include/defines_name/project.conf b/tests/format/include/defines_name/project.conf
new file mode 100644
index 0000000..a7791a4
--- /dev/null
+++ b/tests/format/include/defines_name/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - extra_conf.yml
diff --git a/tests/format/include/file/element.bst b/tests/format/include/file/element.bst
new file mode 100644
index 0000000..4d7f702
--- /dev/null
+++ b/tests/format/include/file/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/file/extra_conf.yml b/tests/format/include/file/extra_conf.yml
new file mode 100644
index 0000000..404ecd6
--- /dev/null
+++ b/tests/format/include/file/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+ included: 'True'
diff --git a/tests/format/include/file/project.conf b/tests/format/include/file/project.conf
new file mode 100644
index 0000000..a7791a4
--- /dev/null
+++ b/tests/format/include/file/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - extra_conf.yml
diff --git a/tests/format/include/file_with_subproject/element.bst b/tests/format/include/file_with_subproject/element.bst
new file mode 100644
index 0000000..4d7f702
--- /dev/null
+++ b/tests/format/include/file_with_subproject/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/file_with_subproject/extra_conf.yml b/tests/format/include/file_with_subproject/extra_conf.yml
new file mode 100644
index 0000000..404ecd6
--- /dev/null
+++ b/tests/format/include/file_with_subproject/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+ included: 'True'
diff --git a/tests/format/include/file_with_subproject/project.bst b/tests/format/include/file_with_subproject/project.bst
new file mode 100644
index 0000000..4836c5f
--- /dev/null
+++ b/tests/format/include/file_with_subproject/project.bst
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - junction.bst:extra_conf.yml
diff --git a/tests/format/include/file_with_subproject/project.conf b/tests/format/include/file_with_subproject/project.conf
new file mode 100644
index 0000000..a7791a4
--- /dev/null
+++ b/tests/format/include/file_with_subproject/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - extra_conf.yml
diff --git a/tests/format/include/file_with_subproject/subproject/project.conf b/tests/format/include/file_with_subproject/subproject/project.conf
new file mode 100644
index 0000000..7a66554
--- /dev/null
+++ b/tests/format/include/file_with_subproject/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/junction/element.bst b/tests/format/include/junction/element.bst
new file mode 100644
index 0000000..4d7f702
--- /dev/null
+++ b/tests/format/include/junction/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/junction/project.conf b/tests/format/include/junction/project.conf
new file mode 100644
index 0000000..4836c5f
--- /dev/null
+++ b/tests/format/include/junction/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - junction.bst:extra_conf.yml
diff --git a/tests/format/include/junction/subproject/extra_conf.yml b/tests/format/include/junction/subproject/extra_conf.yml
new file mode 100644
index 0000000..404ecd6
--- /dev/null
+++ b/tests/format/include/junction/subproject/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+ included: 'True'
diff --git a/tests/format/include/junction/subproject/project.conf b/tests/format/include/junction/subproject/project.conf
new file mode 100644
index 0000000..7a66554
--- /dev/null
+++ b/tests/format/include/junction/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/options/element.bst b/tests/format/include/options/element.bst
new file mode 100644
index 0000000..4d7f702
--- /dev/null
+++ b/tests/format/include/options/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/options/extra_conf.yml b/tests/format/include/options/extra_conf.yml
new file mode 100644
index 0000000..ad1401e
--- /dev/null
+++ b/tests/format/include/options/extra_conf.yml
@@ -0,0 +1,8 @@
+options:
+ build_arch:
+ type: arch
+ description: Architecture
+ variable: build_arch
+ values:
+ - i586
+ - x86_64
diff --git a/tests/format/include/options/project.conf b/tests/format/include/options/project.conf
new file mode 100644
index 0000000..a7791a4
--- /dev/null
+++ b/tests/format/include/options/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+ - extra_conf.yml
diff --git a/tests/format/include/overrides/element.bst b/tests/format/include/overrides/element.bst
new file mode 100644
index 0000000..4d7f702
--- /dev/null
+++ b/tests/format/include/overrides/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/overrides/extra_conf.yml b/tests/format/include/overrides/extra_conf.yml
new file mode 100644
index 0000000..3cd3530
--- /dev/null
+++ b/tests/format/include/overrides/extra_conf.yml
@@ -0,0 +1,16 @@
+elements:
+ junction:
+ variables:
+ included_override: True
+ manual:
+ variables:
+ manual_included_override: True
+ config:
+ build-commands:
+ (>):
+ - "second"
+
+sources:
+ git:
+ variables:
+ from_included: True
diff --git a/tests/format/include/overrides/project.conf b/tests/format/include/overrides/project.conf
new file mode 100644
index 0000000..9285b9d
--- /dev/null
+++ b/tests/format/include/overrides/project.conf
@@ -0,0 +1,20 @@
+name: test
+
+elements:
+ junction:
+ variables:
+ main_override: True
+ manual:
+ variables:
+ manual_main_override: True
+ config:
+ build-commands:
+ - "first"
+
+sources:
+ git:
+ variables:
+ from_main: True
+
+(@):
+ - extra_conf.yml
diff --git a/tests/format/include/overrides/subproject/project.conf b/tests/format/include/overrides/subproject/project.conf
new file mode 100644
index 0000000..7a66554
--- /dev/null
+++ b/tests/format/include/overrides/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/sub-include/element.bst b/tests/format/include/sub-include/element.bst
new file mode 100644
index 0000000..4d7f702
--- /dev/null
+++ b/tests/format/include/sub-include/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/sub-include/manual_conf.yml b/tests/format/include/sub-include/manual_conf.yml
new file mode 100644
index 0000000..9c2c0dd
--- /dev/null
+++ b/tests/format/include/sub-include/manual_conf.yml
@@ -0,0 +1,2 @@
+variables:
+ included: True
diff --git a/tests/format/include/sub-include/project.conf b/tests/format/include/sub-include/project.conf
new file mode 100644
index 0000000..7f7df84
--- /dev/null
+++ b/tests/format/include/sub-include/project.conf
@@ -0,0 +1,6 @@
+name: test
+
+elements:
+ manual:
+ (@):
+ - manual_conf.yml