You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 07:47:41 UTC
[buildstream] 02/02: Continue lazy provenance info
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch test_pyyaml
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 97c7151f61c55739e65720f50910b880d5c03bd5
Author: Gökçen Nurlu <gn...@bloomberg.net>
AuthorDate: Mon Feb 11 10:47:16 2019 +0000
Continue lazy provenance info
---
buildstream/_artifactcache.py | 7 ++-
buildstream/_includes.py | 2 +-
buildstream/_loader/loadelement.py | 2 +-
buildstream/_loader/loader.py | 4 +-
buildstream/_options/optionpool.py | 4 +-
buildstream/_project.py | 8 ++-
buildstream/_variables.py | 5 +-
buildstream/_yaml.py | 100 +++++++++++++++++++++++++++----------
buildstream/element.py | 7 +--
buildstream/source.py | 6 ++-
10 files changed, 100 insertions(+), 45 deletions(-)
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 48ab278..97e4a1f 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -195,10 +195,13 @@ class ArtifactCache():
artifacts = config_node.get('artifacts', [])
if isinstance(artifacts, Mapping):
- cache_specs.append(ArtifactCacheSpec._new_from_config_node(artifacts, basedir))
+ cache_specs.append(ArtifactCacheSpec._new_from_config_node(
+ _yaml.BstNode(config_node.bst_filename, artifacts, config_node.path + ('artifacts',)), basedir)
+ )
elif isinstance(artifacts, list):
for spec_node in artifacts:
- cache_specs.append(ArtifactCacheSpec._new_from_config_node(spec_node, basedir))
+ cache_specs.append(ArtifactCacheSpec._new_from_config_node(
+ _yaml.BstNode(config_node.bst_filename, spec_node, config_node.path + ('artifacts',)), basedir))
else:
provenance = _yaml.node_get_provenance(config_node, key='artifacts')
raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
diff --git a/buildstream/_includes.py b/buildstream/_includes.py
index df14c9f..17e5e4b 100644
--- a/buildstream/_includes.py
+++ b/buildstream/_includes.py
@@ -120,7 +120,7 @@ class Includes:
included=set(),
current_loader=None,
only_local=False):
- if isinstance(value, Mapping):
+ if isinstance(value, _yaml.BstNode):
self.process(value,
included=included,
current_loader=current_loader,
diff --git a/buildstream/_loader/loadelement.py b/buildstream/_loader/loadelement.py
index 11a3776..980392a 100644
--- a/buildstream/_loader/loadelement.py
+++ b/buildstream/_loader/loadelement.py
@@ -184,6 +184,6 @@ def _extract_depends_from_node(node, *, key=None):
# Now delete the field, we dont want it anymore
if key in node:
- del node[key]
+ del node.yaml_node[key]
return output_deps
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index 13761fb..1790972 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -453,12 +453,12 @@ class Loader():
for i in range(len(sources)):
source = _yaml.node_get(node, Mapping, Symbol.SOURCES, indices=[i])
kind = _yaml.node_get(source, str, Symbol.KIND)
- del source[Symbol.KIND]
+ del source.yaml_node[Symbol.KIND]
# Directory is optional
directory = _yaml.node_get(source, str, Symbol.DIRECTORY, default_value=None)
if directory:
- del source[Symbol.DIRECTORY]
+ del source.yaml_node[Symbol.DIRECTORY]
index = sources.index(source)
meta_source = MetaSource(element_name, index, element_kind, kind, source, directory)
diff --git a/buildstream/_options/optionpool.py b/buildstream/_options/optionpool.py
index 3132af5..45da7fc 100644
--- a/buildstream/_options/optionpool.py
+++ b/buildstream/_options/optionpool.py
@@ -187,7 +187,7 @@ class OptionPool():
# and process any indirectly nested conditionals.
#
for _, value in _yaml.node_items(node):
- if isinstance(value, Mapping):
+ if isinstance(value, _yaml.BstNode):
self.process_node(value)
elif isinstance(value, list):
self._process_list(value)
@@ -238,7 +238,7 @@ class OptionPool():
#
def _process_list(self, values):
for value in values:
- if isinstance(value, Mapping):
+ if isinstance(value, _yaml.BstNode):
self.process_node(value)
elif isinstance(value, list):
self._process_list(value)
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 3ec141d..07db1b3 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -268,8 +268,10 @@ class Project():
#
def create_source(self, meta, *, first_pass=False):
if first_pass:
+ print("FIRST PASS", meta.config)
return self.first_pass_config.source_factory.create(self._context, self, meta)
else:
+ print("SECOND", meta.config)
return self.config.source_factory.create(self._context, self, meta)
# get_alias_uri()
@@ -456,6 +458,8 @@ class Project():
raise
pre_config_node = _yaml.node_copy(self._default_config_node)
+ print(pre_config_node.bst_filename)
+ print(self._project_conf.bst_filename)
_yaml.composite(pre_config_node, self._project_conf)
# Assert project's format version early, before validating toplevel keys
@@ -642,8 +646,8 @@ class Project():
# assertion after.
output.element_overrides = _yaml.node_get(config, Mapping, 'elements', default_value={})
output.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
- config.pop('elements', None)
- config.pop('sources', None)
+ config.yaml_node.pop('elements', None)
+ config.yaml_node.pop('sources', None)
_yaml.node_final_assertions(config)
self._load_plugin_factories(config, output)
diff --git a/buildstream/_variables.py b/buildstream/_variables.py
index 95b80cc..c329475 100644
--- a/buildstream/_variables.py
+++ b/buildstream/_variables.py
@@ -43,7 +43,6 @@ _VARIABLE_MATCH = r'\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}'
class Variables():
def __init__(self, node):
-
self.original = node
self.variables = self._resolve(node)
@@ -147,8 +146,8 @@ class Variables():
unmatched += item_unmatched
# Carry over provenance
- resolved[_yaml.PROVENANCE_KEY] = variables[_yaml.PROVENANCE_KEY]
- return (resolved, unmatched)
+ # resolved[_yaml.PROVENANCE_KEY] = variables[_yaml.PROVENANCE_KEY]
+ return (_yaml.BstNode(variables.bst_filename, resolved, variables.path), unmatched)
# Resolve it until it's resolved or broken
#
diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py
index 9668867..4212265 100644
--- a/buildstream/_yaml.py
+++ b/buildstream/_yaml.py
@@ -178,6 +178,30 @@ class CompositeTypeError(CompositeError):
self.actual_type = actual_type
+class BstNode(collections.abc.Mapping):
+ __slots__ = ('bst_filename', 'yaml_node', 'path')
+
+ def __init__(self, bst_filename, yaml_node, path):
+ self.yaml_node = yaml_node
+ self.bst_filename = bst_filename
+ self.path = path
+
+ def __getitem__(self, key):
+ return self.yaml_node.__getitem__(key)
+
+ def __setitem__(self, k, v):
+ self.yaml_node.__setitem__(k,v)
+
+ def __iter__(self):
+ return self.yaml_node.__iter__()
+
+ def __len__(self):
+ return self.yaml_node.__len__()
+
+ def __str__(self):
+ return str((self.yaml_node, self.bst_filename, self.path))
+
+
# Loads a dictionary from some YAML
#
# Args:
@@ -240,7 +264,8 @@ def load_data(data, file=None, copy_tree=False):
.format(type(contents).__name__, file.name))
# return node_decorated_copy(file, contents, copy_tree=copy_tree)
- return contents
+ # return contents
+ return BstNode(file.name, contents, tuple())
# Dumps a previously loaded YAML node to a file
@@ -329,14 +354,15 @@ def node_decorate_list(filename, target, source, toplevel):
#
def node_get_provenance(node, key=None, indices=None):
- provenance = node.get(PROVENANCE_KEY)
- if provenance and key:
- provenance = provenance.members.get(key)
- if provenance and indices is not None:
- for index in indices:
- provenance = provenance.elements[index]
+ # provenance = node.get(PROVENANCE_KEY)
+ # print(node, key, indices)
+ # if provenance and key:
+ # provenance = provenance.members.get(key)
+ # if provenance and indices is not None:
+ # for index in indices:
+ # provenance = provenance.elements[index]
- return provenance
+ return node.bst_filename, node.path
# A sentinel to be used as a default argument for functions that need
@@ -367,17 +393,18 @@ _sentinel = object()
# Note:
# Returned strings are stripped of leading and trailing whitespace
#
-def node_get(node, expected_type, key, indices=None, *, default_value=_sentinel, allow_none=False):
+def node_get(bst_node, expected_type, key, indices=None, *, default_value=_sentinel, allow_none=False):
+ node = bst_node.yaml_node
value = node.get(key, default_value)
- provenance = node_get_provenance(node)
if value is _sentinel:
+ provenance = node_get_provenance(bst_node)
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}: Dictionary did not contain expected key '{}'".format(provenance, key))
path = key
if indices is not None:
# Implied type check of the element itself
- value = node_get(node, list, key)
+ value = node_get(bst_node, list, key)
for index in indices:
value = value[index]
path += '[{:d}]'.format(index)
@@ -406,7 +433,7 @@ def node_get(node, expected_type, key, indices=None, *, default_value=_sentinel,
else:
raise ValueError()
except (ValueError, TypeError):
- provenance = node_get_provenance(node, key=key, indices=indices)
+ provenance = node_get_provenance(bst_node, key=key, indices=indices)
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}: Value of '{}' is not of the expected type '{}'"
.format(provenance, path, expected_type.__name__))
@@ -415,6 +442,8 @@ def node_get(node, expected_type, key, indices=None, *, default_value=_sentinel,
if isinstance(value, str):
value = value.strip()
+ if expected_type == collections.abc.Mapping:
+ return BstNode(bst_node.bst_filename, value, bst_node.path + (key,))
return value
@@ -530,7 +559,10 @@ def node_items(node):
for key, value in node.items():
if key == PROVENANCE_KEY:
continue
- yield (key, value)
+ if isinstance(value, collections.abc.Mapping):
+ yield (key, BstNode(node.bst_filename, value, node.path + (key,)))
+ else:
+ yield (key, value)
# Gives a node a dummy provenance, in case of compositing dictionaries
@@ -732,21 +764,19 @@ def composite_list(target_node, source_node, key):
target_value = target_node.get(key)
source_value = source_node[key]
- target_key_provenance = node_get_provenance(target_node, key)
- source_key_provenance = node_get_provenance(source_node, key)
-
# Whenever a literal list is encountered in the source, it
# overwrites the target values and provenance completely.
#
if isinstance(source_value, list):
- source_provenance = node_get_provenance(source_node)
- target_provenance = node_get_provenance(target_node)
+
# Assert target type
if not (target_value is None or
isinstance(target_value, list) or
is_composite_list(target_value)):
+ target_key_provenance = node_get_provenance(target_node, key)
+ source_key_provenance = node_get_provenance(source_node, key)
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}: List cannot overwrite value at: {}"
.format(source_key_provenance, target_key_provenance))
@@ -837,6 +867,9 @@ def composite_dict(target, source, path=None):
# target_provenance = ensure_provenance(target)
# source_provenance = ensure_provenance(source)
+ #source = bstnode_source.yaml_node
+ #target = bstnode_target.yaml_node
+
for key, source_value in node_items(source):
# Track the full path of keys, only for raising CompositeError
@@ -859,9 +892,9 @@ def composite_dict(target, source, path=None):
target[key] = target_value
# Give the new dict provenance
- value_provenance = source_value.get(PROVENANCE_KEY)
- if value_provenance:
- target_value[PROVENANCE_KEY] = value_provenance.clone()
+ # value_provenance = source_value.get(PROVENANCE_KEY)
+ # if value_provenance:
+ # target_value[PROVENANCE_KEY] = value_provenance.clone()
# Add a new provenance member element to the containing dict
# target_provenance.members[key] = source_provenance.members[key]
@@ -890,13 +923,17 @@ def composite_dict(target, source, path=None):
# Like composite_dict(), but raises an all purpose LoadError for convenience
#
def composite(target, source):
+ print(f"Merging\n\t{source.bst_filename} into\n\t{target.bst_filename}\n")
assert hasattr(source, 'get')
- source_provenance = node_get_provenance(source)
try:
composite_dict(target, source)
+ src = source.bst_filename if isinstance(source.bst_filename, tuple) else (source.bst_filename,)
+ trg = target.bst_filename if isinstance(target.bst_filename, tuple) else (target.bst_filename,)
+ target.bst_filename = trg + src
except CompositeTypeError as e:
error_prefix = ""
+ source_provenance = node_get_provenance(source)
if source_provenance:
error_prefix = "{}: ".format(source_provenance)
raise LoadError(LoadErrorReason.ILLEGAL_COMPOSITE,
@@ -926,13 +963,15 @@ RoundTripRepresenter.add_representer(SanitizedDict,
# Only dicts are ordered, list elements are left in order.
#
def node_sanitize(node):
+ if isinstance(node, BstNode):
+ node = node.yaml_node
if isinstance(node, collections.abc.Mapping):
result = SanitizedDict()
- key_list = [key for key, _ in node_items(node)]
- for key in sorted(key_list):
+ # key_list = [key for key, _ in node_items(node)]
+ for key in sorted(node.keys()):
result[key] = node_sanitize(node[key])
return result
@@ -957,11 +996,12 @@ def node_sanitize(node):
# LoadError: In the case that the specified node contained
# one or more invalid keys
#
-def node_validate(node, valid_keys):
+def node_validate(bst_node, valid_keys):
+ node = bst_node.yaml_node
# Probably the fastest way to do this: https://stackoverflow.com/a/23062482
valid_keys = set(valid_keys)
- valid_keys.add(PROVENANCE_KEY)
+ # valid_keys.add(PROVENANCE_KEY)
invalid = next((key for key in node if key not in valid_keys), None)
if invalid:
@@ -1069,6 +1109,9 @@ class ChainMap(collections.ChainMap):
def node_chain_copy(source):
+ return BstNode(
+ source.bst_filename, deepcopy(source.yaml_node), source.path
+ )
copy = ChainMap({}, source)
for key, value in source.items():
if isinstance(value, collections.abc.Mapping):
@@ -1096,7 +1139,10 @@ def list_chain_copy(source):
return copy
-def node_copy(source):
+def node_copy(bst_node):
+ return BstNode(
+ bst_node.bst_filename, deepcopy(bst_node.yaml_node), bst_node.path
+ )
copy = {}
for key, value in source.items():
if isinstance(value, collections.abc.Mapping):
diff --git a/buildstream/element.py b/buildstream/element.py
index a243826..7a7eb2b 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -2303,7 +2303,7 @@ class Element(Plugin):
if not self.__defaults_set:
# Load the plugin's accompanying .yaml file if one was provided
- defaults = {}
+ defaults = _yaml.BstNode('', {}, tuple())
try:
defaults = _yaml.load(plugin_conf, os.path.basename(plugin_conf))
except LoadError as e:
@@ -2389,8 +2389,9 @@ class Element(Plugin):
_yaml.node_final_assertions(variables)
for var in ('project-name', 'element-name', 'max-jobs'):
- provenance = _yaml.node_get_provenance(variables, var)
- if provenance and provenance.filename != '':
+ #if provenance and provenance.filename != '':
+ if var in meta.variables.yaml_node:
+ provenance = _yaml.node_get_provenance(variables, var)
raise LoadError(LoadErrorReason.PROTECTED_VARIABLE_REDEFINED,
"{}: invalid redefinition of protected variable '{}'"
.format(provenance, var))
diff --git a/buildstream/source.py b/buildstream/source.py
index 9e9bad7..eaecec1 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -254,7 +254,7 @@ class Source(Plugin):
All Sources derive from this class, this interface defines how
the core will be interacting with Sources.
"""
- __defaults = {} # The defaults from the project
+ __defaults = _yaml.BstNode('', {}, tuple()) # The defaults from the project
__defaults_set = False # Flag, in case there are not defaults at all
BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False
@@ -1117,7 +1117,7 @@ class Source(Plugin):
sources = project.first_pass_config.source_overrides
else:
sources = project.source_overrides
- type(self).__defaults = sources.get(self.get_kind(), {})
+ type(self).__defaults = sources.get(self.get_kind(), _yaml.BstNode('', {}, tuple()))
type(self).__defaults_set = True
# This will resolve the final configuration to be handed
@@ -1127,6 +1127,8 @@ class Source(Plugin):
config = _yaml.node_get(self.__defaults, Mapping, 'config', default_value={})
config = _yaml.node_chain_copy(config)
+ print("copied config:", config)
+ print("meta config:", meta.config)
_yaml.composite(config, meta.config)
_yaml.node_final_assertions(config)