You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:53:09 UTC
[buildstream] branch test_pyyaml created (now 97c7151)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch test_pyyaml
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 97c7151 Continue lazy provenance info
This branch includes the following new commits:
new 4ed12f6 test
new 97c7151 Continue lazy provenance info
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/02: test
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch test_pyyaml
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4ed12f6101547786863c2d8deb68d1c50c005d66
Author: Gökçen Nurlu <gn...@bloomberg.net>
AuthorDate: Thu Jan 31 12:32:51 2019 +0000
test
---
buildstream/_loader/loadelement.py | 3 ++-
buildstream/_yaml.py | 24 ++++++++++++++----------
2 files changed, 16 insertions(+), 11 deletions(-)
diff --git a/buildstream/_loader/loadelement.py b/buildstream/_loader/loadelement.py
index 1c520f6..11a3776 100644
--- a/buildstream/_loader/loadelement.py
+++ b/buildstream/_loader/loadelement.py
@@ -183,6 +183,7 @@ def _extract_depends_from_node(node, *, key=None):
output_deps.append(dependency)
# Now delete the field, we dont want it anymore
- del node[key]
+ if key in node:
+ del node[key]
return output_deps
diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py
index 753254e..9668867 100644
--- a/buildstream/_yaml.py
+++ b/buildstream/_yaml.py
@@ -24,11 +24,13 @@ from copy import deepcopy
from contextlib import ExitStack
from pathlib import Path
-from ruamel import yaml
+# from ruamel import yaml as ryaml
from ruamel.yaml.representer import SafeRepresenter, RoundTripRepresenter
from ruamel.yaml.constructor import RoundTripConstructor
from ._exceptions import LoadError, LoadErrorReason
+import yaml
+
# This overrides the ruamel constructor to treat everything as a string
RoundTripConstructor.add_constructor(u'tag:yaml.org,2002:int', RoundTripConstructor.construct_yaml_str)
RoundTripConstructor.add_constructor(u'tag:yaml.org,2002:float', RoundTripConstructor.construct_yaml_str)
@@ -222,7 +224,8 @@ def load(filename, shortname=None, copy_tree=False, *, project=None, yaml_cache=
def load_data(data, file=None, copy_tree=False):
try:
- contents = yaml.load(data, yaml.loader.RoundTripLoader, preserve_quotes=True)
+ # contents = ryaml.load(data, ryaml.loader.RoundTripLoader, preserve_quotes=True)
+ contents = yaml.load(data, Loader=yaml.CLoader)
except (yaml.scanner.ScannerError, yaml.composer.ComposerError, yaml.parser.ParserError) as e:
raise LoadError(LoadErrorReason.INVALID_YAML,
"Malformed YAML:\n\n{}\n\n{}\n".format(e.problem, e.problem_mark)) from e
@@ -236,7 +239,8 @@ def load_data(data, file=None, copy_tree=False):
"YAML file has content of type '{}' instead of expected type 'dict': {}"
.format(type(contents).__name__, file.name))
- return node_decorated_copy(file, contents, copy_tree=copy_tree)
+ # return node_decorated_copy(file, contents, copy_tree=copy_tree)
+ return contents
# Dumps a previously loaded YAML node to a file
@@ -273,7 +277,7 @@ def node_decorated_copy(filename, toplevel, copy_tree=False):
else:
result = toplevel
- node_decorate_dict(filename, result, toplevel, toplevel)
+ # node_decorate_dict(filename, result, toplevel, toplevel)
return result
@@ -554,8 +558,8 @@ def is_ruamel_str(value):
if isinstance(value, str):
return True
- elif isinstance(value, yaml.scalarstring.ScalarString):
- return True
+ #elif isinstance(value, yaml.scalarstring.ScalarString):
+ # return True
return False
@@ -830,8 +834,8 @@ def composite_list(target_node, source_node, key):
# configurations.
#
def composite_dict(target, source, path=None):
- target_provenance = ensure_provenance(target)
- source_provenance = ensure_provenance(source)
+ # target_provenance = ensure_provenance(target)
+ # source_provenance = ensure_provenance(source)
for key, source_value in node_items(source):
@@ -860,7 +864,7 @@ def composite_dict(target, source, path=None):
target_value[PROVENANCE_KEY] = value_provenance.clone()
# Add a new provenance member element to the containing dict
- target_provenance.members[key] = source_provenance.members[key]
+ # target_provenance.members[key] = source_provenance.members[key]
if not isinstance(target_value, collections.abc.Mapping):
raise CompositeTypeError(thispath, type(target_value), type(source_value))
@@ -879,7 +883,7 @@ def composite_dict(target, source, path=None):
raise CompositeTypeError(thispath, type(target_value), type(source_value))
# Overwrite simple values, lists and mappings have already been handled
- target_provenance.members[key] = source_provenance.members[key].clone()
+ # target_provenance.members[key] = source_provenance.members[key].clone()
target[key] = source_value
[buildstream] 02/02: Continue lazy provenance info
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch test_pyyaml
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 97c7151f61c55739e65720f50910b880d5c03bd5
Author: Gökçen Nurlu <gn...@bloomberg.net>
AuthorDate: Mon Feb 11 10:47:16 2019 +0000
Continue lazy provenance info
---
buildstream/_artifactcache.py | 7 ++-
buildstream/_includes.py | 2 +-
buildstream/_loader/loadelement.py | 2 +-
buildstream/_loader/loader.py | 4 +-
buildstream/_options/optionpool.py | 4 +-
buildstream/_project.py | 8 ++-
buildstream/_variables.py | 5 +-
buildstream/_yaml.py | 100 +++++++++++++++++++++++++++----------
buildstream/element.py | 7 +--
buildstream/source.py | 6 ++-
10 files changed, 100 insertions(+), 45 deletions(-)
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 48ab278..97e4a1f 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -195,10 +195,13 @@ class ArtifactCache():
artifacts = config_node.get('artifacts', [])
if isinstance(artifacts, Mapping):
- cache_specs.append(ArtifactCacheSpec._new_from_config_node(artifacts, basedir))
+ cache_specs.append(ArtifactCacheSpec._new_from_config_node(
+ _yaml.BstNode(config_node.bst_filename, artifacts, config_node.path + ('artifacts',)), basedir)
+ )
elif isinstance(artifacts, list):
for spec_node in artifacts:
- cache_specs.append(ArtifactCacheSpec._new_from_config_node(spec_node, basedir))
+ cache_specs.append(ArtifactCacheSpec._new_from_config_node(
+ _yaml.BstNode(config_node.bst_filename, spec_node, config_node.path + ('artifacts',)), basedir))
else:
provenance = _yaml.node_get_provenance(config_node, key='artifacts')
raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
diff --git a/buildstream/_includes.py b/buildstream/_includes.py
index df14c9f..17e5e4b 100644
--- a/buildstream/_includes.py
+++ b/buildstream/_includes.py
@@ -120,7 +120,7 @@ class Includes:
included=set(),
current_loader=None,
only_local=False):
- if isinstance(value, Mapping):
+ if isinstance(value, _yaml.BstNode):
self.process(value,
included=included,
current_loader=current_loader,
diff --git a/buildstream/_loader/loadelement.py b/buildstream/_loader/loadelement.py
index 11a3776..980392a 100644
--- a/buildstream/_loader/loadelement.py
+++ b/buildstream/_loader/loadelement.py
@@ -184,6 +184,6 @@ def _extract_depends_from_node(node, *, key=None):
# Now delete the field, we dont want it anymore
if key in node:
- del node[key]
+ del node.yaml_node[key]
return output_deps
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index 13761fb..1790972 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -453,12 +453,12 @@ class Loader():
for i in range(len(sources)):
source = _yaml.node_get(node, Mapping, Symbol.SOURCES, indices=[i])
kind = _yaml.node_get(source, str, Symbol.KIND)
- del source[Symbol.KIND]
+ del source.yaml_node[Symbol.KIND]
# Directory is optional
directory = _yaml.node_get(source, str, Symbol.DIRECTORY, default_value=None)
if directory:
- del source[Symbol.DIRECTORY]
+ del source.yaml_node[Symbol.DIRECTORY]
index = sources.index(source)
meta_source = MetaSource(element_name, index, element_kind, kind, source, directory)
diff --git a/buildstream/_options/optionpool.py b/buildstream/_options/optionpool.py
index 3132af5..45da7fc 100644
--- a/buildstream/_options/optionpool.py
+++ b/buildstream/_options/optionpool.py
@@ -187,7 +187,7 @@ class OptionPool():
# and process any indirectly nested conditionals.
#
for _, value in _yaml.node_items(node):
- if isinstance(value, Mapping):
+ if isinstance(value, _yaml.BstNode):
self.process_node(value)
elif isinstance(value, list):
self._process_list(value)
@@ -238,7 +238,7 @@ class OptionPool():
#
def _process_list(self, values):
for value in values:
- if isinstance(value, Mapping):
+ if isinstance(value, _yaml.BstNode):
self.process_node(value)
elif isinstance(value, list):
self._process_list(value)
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 3ec141d..07db1b3 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -268,8 +268,10 @@ class Project():
#
def create_source(self, meta, *, first_pass=False):
if first_pass:
+ print("FIRST PASS", meta.config)
return self.first_pass_config.source_factory.create(self._context, self, meta)
else:
+ print("SECOND", meta.config)
return self.config.source_factory.create(self._context, self, meta)
# get_alias_uri()
@@ -456,6 +458,8 @@ class Project():
raise
pre_config_node = _yaml.node_copy(self._default_config_node)
+ print(pre_config_node.bst_filename)
+ print(self._project_conf.bst_filename)
_yaml.composite(pre_config_node, self._project_conf)
# Assert project's format version early, before validating toplevel keys
@@ -642,8 +646,8 @@ class Project():
# assertion after.
output.element_overrides = _yaml.node_get(config, Mapping, 'elements', default_value={})
output.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
- config.pop('elements', None)
- config.pop('sources', None)
+ config.yaml_node.pop('elements', None)
+ config.yaml_node.pop('sources', None)
_yaml.node_final_assertions(config)
self._load_plugin_factories(config, output)
diff --git a/buildstream/_variables.py b/buildstream/_variables.py
index 95b80cc..c329475 100644
--- a/buildstream/_variables.py
+++ b/buildstream/_variables.py
@@ -43,7 +43,6 @@ _VARIABLE_MATCH = r'\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}'
class Variables():
def __init__(self, node):
-
self.original = node
self.variables = self._resolve(node)
@@ -147,8 +146,8 @@ class Variables():
unmatched += item_unmatched
# Carry over provenance
- resolved[_yaml.PROVENANCE_KEY] = variables[_yaml.PROVENANCE_KEY]
- return (resolved, unmatched)
+ # resolved[_yaml.PROVENANCE_KEY] = variables[_yaml.PROVENANCE_KEY]
+ return (_yaml.BstNode(variables.bst_filename, resolved, variables.path), unmatched)
# Resolve it until it's resolved or broken
#
diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py
index 9668867..4212265 100644
--- a/buildstream/_yaml.py
+++ b/buildstream/_yaml.py
@@ -178,6 +178,30 @@ class CompositeTypeError(CompositeError):
self.actual_type = actual_type
+class BstNode(collections.abc.Mapping):
+ __slots__ = ('bst_filename', 'yaml_node', 'path')
+
+ def __init__(self, bst_filename, yaml_node, path):
+ self.yaml_node = yaml_node
+ self.bst_filename = bst_filename
+ self.path = path
+
+ def __getitem__(self, key):
+ return self.yaml_node.__getitem__(key)
+
+ def __setitem__(self, k, v):
+ self.yaml_node.__setitem__(k,v)
+
+ def __iter__(self):
+ return self.yaml_node.__iter__()
+
+ def __len__(self):
+ return self.yaml_node.__len__()
+
+ def __str__(self):
+ return str((self.yaml_node, self.bst_filename, self.path))
+
+
# Loads a dictionary from some YAML
#
# Args:
@@ -240,7 +264,8 @@ def load_data(data, file=None, copy_tree=False):
.format(type(contents).__name__, file.name))
# return node_decorated_copy(file, contents, copy_tree=copy_tree)
- return contents
+ # return contents
+ return BstNode(file.name, contents, tuple())
# Dumps a previously loaded YAML node to a file
@@ -329,14 +354,15 @@ def node_decorate_list(filename, target, source, toplevel):
#
def node_get_provenance(node, key=None, indices=None):
- provenance = node.get(PROVENANCE_KEY)
- if provenance and key:
- provenance = provenance.members.get(key)
- if provenance and indices is not None:
- for index in indices:
- provenance = provenance.elements[index]
+ # provenance = node.get(PROVENANCE_KEY)
+ # print(node, key, indices)
+ # if provenance and key:
+ # provenance = provenance.members.get(key)
+ # if provenance and indices is not None:
+ # for index in indices:
+ # provenance = provenance.elements[index]
- return provenance
+ return node.bst_filename, node.path
# A sentinel to be used as a default argument for functions that need
@@ -367,17 +393,18 @@ _sentinel = object()
# Note:
# Returned strings are stripped of leading and trailing whitespace
#
-def node_get(node, expected_type, key, indices=None, *, default_value=_sentinel, allow_none=False):
+def node_get(bst_node, expected_type, key, indices=None, *, default_value=_sentinel, allow_none=False):
+ node = bst_node.yaml_node
value = node.get(key, default_value)
- provenance = node_get_provenance(node)
if value is _sentinel:
+ provenance = node_get_provenance(bst_node)
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}: Dictionary did not contain expected key '{}'".format(provenance, key))
path = key
if indices is not None:
# Implied type check of the element itself
- value = node_get(node, list, key)
+ value = node_get(bst_node, list, key)
for index in indices:
value = value[index]
path += '[{:d}]'.format(index)
@@ -406,7 +433,7 @@ def node_get(node, expected_type, key, indices=None, *, default_value=_sentinel,
else:
raise ValueError()
except (ValueError, TypeError):
- provenance = node_get_provenance(node, key=key, indices=indices)
+ provenance = node_get_provenance(bst_node, key=key, indices=indices)
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}: Value of '{}' is not of the expected type '{}'"
.format(provenance, path, expected_type.__name__))
@@ -415,6 +442,8 @@ def node_get(node, expected_type, key, indices=None, *, default_value=_sentinel,
if isinstance(value, str):
value = value.strip()
+ if expected_type == collections.abc.Mapping:
+ return BstNode(bst_node.bst_filename, value, bst_node.path + (key,))
return value
@@ -530,7 +559,10 @@ def node_items(node):
for key, value in node.items():
if key == PROVENANCE_KEY:
continue
- yield (key, value)
+ if isinstance(value, collections.abc.Mapping):
+ yield (key, BstNode(node.bst_filename, value, node.path + (key,)))
+ else:
+ yield (key, value)
# Gives a node a dummy provenance, in case of compositing dictionaries
@@ -732,21 +764,19 @@ def composite_list(target_node, source_node, key):
target_value = target_node.get(key)
source_value = source_node[key]
- target_key_provenance = node_get_provenance(target_node, key)
- source_key_provenance = node_get_provenance(source_node, key)
-
# Whenever a literal list is encountered in the source, it
# overwrites the target values and provenance completely.
#
if isinstance(source_value, list):
- source_provenance = node_get_provenance(source_node)
- target_provenance = node_get_provenance(target_node)
+
# Assert target type
if not (target_value is None or
isinstance(target_value, list) or
is_composite_list(target_value)):
+ target_key_provenance = node_get_provenance(target_node, key)
+ source_key_provenance = node_get_provenance(source_node, key)
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}: List cannot overwrite value at: {}"
.format(source_key_provenance, target_key_provenance))
@@ -837,6 +867,9 @@ def composite_dict(target, source, path=None):
# target_provenance = ensure_provenance(target)
# source_provenance = ensure_provenance(source)
+ #source = bstnode_source.yaml_node
+ #target = bstnode_target.yaml_node
+
for key, source_value in node_items(source):
# Track the full path of keys, only for raising CompositeError
@@ -859,9 +892,9 @@ def composite_dict(target, source, path=None):
target[key] = target_value
# Give the new dict provenance
- value_provenance = source_value.get(PROVENANCE_KEY)
- if value_provenance:
- target_value[PROVENANCE_KEY] = value_provenance.clone()
+ # value_provenance = source_value.get(PROVENANCE_KEY)
+ # if value_provenance:
+ # target_value[PROVENANCE_KEY] = value_provenance.clone()
# Add a new provenance member element to the containing dict
# target_provenance.members[key] = source_provenance.members[key]
@@ -890,13 +923,17 @@ def composite_dict(target, source, path=None):
# Like composite_dict(), but raises an all purpose LoadError for convenience
#
def composite(target, source):
+ print(f"Merging\n\t{source.bst_filename} into\n\t{target.bst_filename}\n")
assert hasattr(source, 'get')
- source_provenance = node_get_provenance(source)
try:
composite_dict(target, source)
+ src = source.bst_filename if isinstance(source.bst_filename, tuple) else (source.bst_filename,)
+ trg = target.bst_filename if isinstance(target.bst_filename, tuple) else (target.bst_filename,)
+ target.bst_filename = trg + src
except CompositeTypeError as e:
error_prefix = ""
+ source_provenance = node_get_provenance(source)
if source_provenance:
error_prefix = "{}: ".format(source_provenance)
raise LoadError(LoadErrorReason.ILLEGAL_COMPOSITE,
@@ -926,13 +963,15 @@ RoundTripRepresenter.add_representer(SanitizedDict,
# Only dicts are ordered, list elements are left in order.
#
def node_sanitize(node):
+ if isinstance(node, BstNode):
+ node = node.yaml_node
if isinstance(node, collections.abc.Mapping):
result = SanitizedDict()
- key_list = [key for key, _ in node_items(node)]
- for key in sorted(key_list):
+ # key_list = [key for key, _ in node_items(node)]
+ for key in sorted(node.keys()):
result[key] = node_sanitize(node[key])
return result
@@ -957,11 +996,12 @@ def node_sanitize(node):
# LoadError: In the case that the specified node contained
# one or more invalid keys
#
-def node_validate(node, valid_keys):
+def node_validate(bst_node, valid_keys):
+ node = bst_node.yaml_node
# Probably the fastest way to do this: https://stackoverflow.com/a/23062482
valid_keys = set(valid_keys)
- valid_keys.add(PROVENANCE_KEY)
+ # valid_keys.add(PROVENANCE_KEY)
invalid = next((key for key in node if key not in valid_keys), None)
if invalid:
@@ -1069,6 +1109,9 @@ class ChainMap(collections.ChainMap):
def node_chain_copy(source):
+ return BstNode(
+ source.bst_filename, deepcopy(source.yaml_node), source.path
+ )
copy = ChainMap({}, source)
for key, value in source.items():
if isinstance(value, collections.abc.Mapping):
@@ -1096,7 +1139,10 @@ def list_chain_copy(source):
return copy
-def node_copy(source):
+def node_copy(bst_node):
+ return BstNode(
+ bst_node.bst_filename, deepcopy(bst_node.yaml_node), bst_node.path
+ )
copy = {}
for key, value in source.items():
if isinstance(value, collections.abc.Mapping):
diff --git a/buildstream/element.py b/buildstream/element.py
index a243826..7a7eb2b 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -2303,7 +2303,7 @@ class Element(Plugin):
if not self.__defaults_set:
# Load the plugin's accompanying .yaml file if one was provided
- defaults = {}
+ defaults = _yaml.BstNode('', {}, tuple())
try:
defaults = _yaml.load(plugin_conf, os.path.basename(plugin_conf))
except LoadError as e:
@@ -2389,8 +2389,9 @@ class Element(Plugin):
_yaml.node_final_assertions(variables)
for var in ('project-name', 'element-name', 'max-jobs'):
- provenance = _yaml.node_get_provenance(variables, var)
- if provenance and provenance.filename != '':
+ #if provenance and provenance.filename != '':
+ if var in meta.variables.yaml_node:
+ provenance = _yaml.node_get_provenance(variables, var)
raise LoadError(LoadErrorReason.PROTECTED_VARIABLE_REDEFINED,
"{}: invalid redefinition of protected variable '{}'"
.format(provenance, var))
diff --git a/buildstream/source.py b/buildstream/source.py
index 9e9bad7..eaecec1 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -254,7 +254,7 @@ class Source(Plugin):
All Sources derive from this class, this interface defines how
the core will be interacting with Sources.
"""
- __defaults = {} # The defaults from the project
+ __defaults = _yaml.BstNode('', {}, tuple()) # The defaults from the project
__defaults_set = False # Flag, in case there are not defaults at all
BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False
@@ -1117,7 +1117,7 @@ class Source(Plugin):
sources = project.first_pass_config.source_overrides
else:
sources = project.source_overrides
- type(self).__defaults = sources.get(self.get_kind(), {})
+ type(self).__defaults = sources.get(self.get_kind(), _yaml.BstNode('', {}, tuple()))
type(self).__defaults_set = True
# This will resolve the final configuration to be handed
@@ -1127,6 +1127,8 @@ class Source(Plugin):
config = _yaml.node_get(self.__defaults, Mapping, 'config', default_value={})
config = _yaml.node_chain_copy(config)
+ print("copied config:", config)
+ print("meta config:", meta.config)
_yaml.composite(config, meta.config)
_yaml.node_final_assertions(config)