You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:23:47 UTC
[buildstream] branch lachlan/pickle-yaml-test-list-composite-park
created (now f9e39bd)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch lachlan/pickle-yaml-test-list-composite-park
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at f9e39bd Add yaml cache test to yaml list composition twice
This branch includes the following new commits:
new 030935f yamlcache/tests: Make yamlcache.open pass a dir in instead of divining its own default
new ae5b554 Add loader for yaml test files cache option
new fa7d0e8 Add yaml cache testing to yaml list composition test
new f9e39bd Add yaml cache test to yaml list composition twice
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/04: yamlcache/tests: Make yamlcache.open pass a
dir in instead of divining its own default
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch lachlan/pickle-yaml-test-list-composite-park
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 030935ff42dbc3be1b17b4c65ce343e56b3a88fc
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Fri Oct 5 11:30:34 2018 +0100
yamlcache/tests: Make yamlcache.open pass a dir in instead of divining its own default
---
buildstream/_loader/loader.py | 8 +++++++-
buildstream/_yamlcache.py | 21 +++++++++------------
tests/frontend/yamlcache.py | 9 +++++----
3 files changed, 21 insertions(+), 17 deletions(-)
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index 7e51efe..984ca87 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -113,7 +113,13 @@ class Loader():
profile_start(Topics.LOAD_PROJECT, target)
junction, name, loader = self._parse_name(target, rewritable, ticker,
fetch_subprojects=fetch_subprojects)
- with YamlCache.open(self._context) as yaml_cache:
+
+ # XXX This will need to be changed to the context's top-level project if this method
+ # is ever used for subprojects
+ top_dir = self.project.directory
+
+ cache_file = YamlCache.get_cache_file(top_dir)
+ with YamlCache.open(self._context, cache_file) as yaml_cache:
loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
deps.append(Dependency(name, junction=junction))
profile_end(Topics.LOAD_PROJECT, target)
diff --git a/buildstream/_yamlcache.py b/buildstream/_yamlcache.py
index 65b3700..12fb940 100644
--- a/buildstream/_yamlcache.py
+++ b/buildstream/_yamlcache.py
@@ -54,8 +54,10 @@ class YamlCache():
self._context = context
# Writes the yaml cache to the specified path.
- def write(self):
- path = self._get_cache_file(self._context)
+ #
+ # Args:
+ # path (str): The path to the cache file.
+ def write(self, path):
parent_dir = os.path.dirname(path)
os.makedirs(parent_dir, exist_ok=True)
with open(path, "wb") as f:
@@ -139,14 +141,14 @@ class YamlCache():
#
# Args:
# context (Context): The context.
+ # cachefile (str): The path to the cache file.
#
# Returns:
# (YamlCache): A YamlCache.
@staticmethod
@contextmanager
- def open(context):
+ def open(context, cachefile):
# Try to load from disk first
- cachefile = YamlCache._get_cache_file(context)
cache = None
if os.path.exists(cachefile):
try:
@@ -155,12 +157,13 @@ class YamlCache():
except pickle.UnpicklingError as e:
sys.stderr.write("Failed to load YamlCache, {}\n".format(e))
+ # Failed to load from disk, create a new one
if not cache:
cache = YamlCache(context)
yield cache
- cache.write()
+ cache.write(cachefile)
# Calculates a key for putting into the cache.
@staticmethod
@@ -170,13 +173,7 @@ class YamlCache():
# Retrieves a path to the yaml cache file.
@staticmethod
- def _get_cache_file(context):
- try:
- toplevel_project = context.get_toplevel_project()
- top_dir = toplevel_project.directory
- except IndexError:
- # Context has no projects, fall back to current directory
- top_dir = os.getcwd()
+ def get_cache_file(top_dir):
return os.path.join(top_dir, ".bst", YAML_CACHE_FILENAME)
diff --git a/tests/frontend/yamlcache.py b/tests/frontend/yamlcache.py
index a8e2cd7..aa1fba9 100644
--- a/tests/frontend/yamlcache.py
+++ b/tests/frontend/yamlcache.py
@@ -52,7 +52,8 @@ def generate_project(tmpdir, ref_storage, with_junction, name="test"):
def with_yamlcache(project_dir):
context = Context()
project = Project(project_dir, context)
- with YamlCache.open(context) as yamlcache:
+ cache_file = YamlCache.get_cache_file(project_dir)
+ with YamlCache.open(context, cache_file) as yamlcache:
yield yamlcache, project
@@ -62,12 +63,12 @@ def yamlcache_key(yamlcache, in_file, copy_tree=False):
return key
-def modified_file(input_file):
+def modified_file(input_file, tmpdir):
with open(input_file) as f:
data = f.read()
assert 'variables' not in data
data += '\nvariables: {modified: True}\n'
- _, temppath = tempfile.mkstemp(text=True)
+ _, temppath = tempfile.mkstemp(dir=tmpdir, text=True)
with open(temppath, 'w') as f:
f.write(data)
@@ -96,7 +97,7 @@ def test_yamlcache_used(cli, tmpdir, ref_storage, with_junction, move_project):
# *Absolutely* horrible cache corruption to check it's being used
# Modifying the data from the cache is fraught with danger,
# so instead I'll load a modified version of the original file
- temppath = modified_file(element_path)
+ temppath = modified_file(element_path, str(tmpdir))
contents = _yaml.load(temppath, copy_tree=False, project=prj)
key = yamlcache_key(yc, element_path)
yc.put(prj, element_path, key, contents)
[buildstream] 04/04: Add yaml cache test to yaml list composition
twice
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch lachlan/pickle-yaml-test-list-composite-park
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f9e39bdc4fbb5b90fbb3348acdb0222e39665840
Author: Lachlan Mackenzie <la...@codethink.co.uk>
AuthorDate: Tue Oct 2 16:23:40 2018 +0100
Add yaml cache test to yaml list composition twice
---
tests/yaml/yaml.py | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/tests/yaml/yaml.py b/tests/yaml/yaml.py
index 692ff01..4a11521 100644
--- a/tests/yaml/yaml.py
+++ b/tests/yaml/yaml.py
@@ -275,6 +275,7 @@ def test_list_deletion(datafiles):
# prov_col: The expected provenance column of "mood"
#
@pytest.mark.datafiles(os.path.join(DATA_DIR))
+@pytest.mark.parametrize('caching', [('raw'), ('cached')])
@pytest.mark.parametrize("filename1,filename2,index,length,mood,prov_file,prov_line,prov_col", [
# Test results of compositing literal list with (>) and then (<)
@@ -331,9 +332,9 @@ def test_list_deletion(datafiles):
('listoverwrite.yaml', 'listprepend.yaml', 2, 4, 'overwrite1', 'listoverwrite.yaml', 5, 10),
('listoverwrite.yaml', 'listprepend.yaml', 3, 4, 'overwrite2', 'listoverwrite.yaml', 7, 10),
])
-def test_list_composition_twice(datafiles, filename1, filename2,
+def test_list_composition_twice(datafiles, tmpdir, filename1, filename2,
index, length, mood,
- prov_file, prov_line, prov_col):
+ prov_file, prov_line, prov_col, caching):
file_base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
file1 = os.path.join(datafiles.dirname, datafiles.basename, filename1)
file2 = os.path.join(datafiles.dirname, datafiles.basename, filename2)
@@ -341,9 +342,9 @@ def test_list_composition_twice(datafiles, filename1, filename2,
#####################
# Round 1 - Fight !
#####################
- base = _yaml.load(file_base, shortname='basics.yaml')
- overlay1 = _yaml.load(file1, shortname=filename1)
- overlay2 = _yaml.load(file2, shortname=filename2)
+ base = load_yaml_file(file_base, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
+ overlay1 = load_yaml_file(file1, cache_path=tmpdir, shortname=filename1, from_cache=caching)
+ overlay2 = load_yaml_file(file2, cache_path=tmpdir, shortname=filename2, from_cache=caching)
_yaml.composite_dict(base, overlay1)
_yaml.composite_dict(base, overlay2)
@@ -358,9 +359,9 @@ def test_list_composition_twice(datafiles, filename1, filename2,
#####################
# Round 2 - Fight !
#####################
- base = _yaml.load(file_base, shortname='basics.yaml')
- overlay1 = _yaml.load(file1, shortname=filename1)
- overlay2 = _yaml.load(file2, shortname=filename2)
+ base = load_yaml_file(file_base, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
+ overlay1 = load_yaml_file(file1, cache_path=tmpdir, shortname=filename1, from_cache=caching)
+ overlay2 = load_yaml_file(file2, cache_path=tmpdir, shortname=filename2, from_cache=caching)
_yaml.composite_dict(overlay1, overlay2)
_yaml.composite_dict(base, overlay1)
[buildstream] 02/04: Add loader for yaml test files cache option
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch lachlan/pickle-yaml-test-list-composite-park
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ae5b554b5762cd29e5818da155721f37a869447d
Author: Lachlan Mackenzie <la...@codethink.co.uk>
AuthorDate: Fri Oct 5 17:32:40 2018 +0100
Add loader for yaml test files cache option
---
tests/yaml/yaml.py | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/tests/yaml/yaml.py b/tests/yaml/yaml.py
index 7817637..7e00f08 100644
--- a/tests/yaml/yaml.py
+++ b/tests/yaml/yaml.py
@@ -1,5 +1,6 @@
import os
import pytest
+import tempfile
from collections import Mapping
from buildstream import _yaml
@@ -150,6 +151,22 @@ def test_composite_preserve_originals(datafiles):
assert(_yaml.node_get(orig_extra, str, 'old') == 'new')
+def load_yaml_file(filename, *, cache_path, shortname=None, from_cache='raw'):
+
+ temppath = tempfile.mkstemp(dir=str(cache_path), text=True)
+ context = Context()
+
+ with YamlCache.open(context, str(temppath)) as yc:
+ if from_cache == 'raw':
+ return _yaml.load(filename, shortname)
+ elif from_cache == 'cached':
+ _yaml.load(filename, shortname)
+
+ return _yaml.load(filename, shortname, yaml_cache=yc)
+ else:
+ assert False
+
+
# Tests for list composition
#
# Each test composits a filename on top of basics.yaml, and tests
[buildstream] 03/04: Add yaml cache testing to yaml list
composition test
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch lachlan/pickle-yaml-test-list-composite-park
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit fa7d0e8883989e841111e14f210dda8209caf9d8
Author: Lachlan Mackenzie <la...@codethink.co.uk>
AuthorDate: Tue Oct 2 15:00:10 2018 +0100
Add yaml cache testing to yaml list composition test
---
tests/yaml/yaml.py | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/tests/yaml/yaml.py b/tests/yaml/yaml.py
index 7e00f08..692ff01 100644
--- a/tests/yaml/yaml.py
+++ b/tests/yaml/yaml.py
@@ -5,6 +5,8 @@ from collections import Mapping
from buildstream import _yaml
from buildstream._exceptions import LoadError, LoadErrorReason
+from buildstream._context import Context
+from buildstream._yamlcache import YamlCache
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
@@ -182,6 +184,7 @@ def load_yaml_file(filename, *, cache_path, shortname=None, from_cache='raw'):
# prov_col: The expected provenance column of "mood"
#
@pytest.mark.datafiles(os.path.join(DATA_DIR))
+@pytest.mark.parametrize('caching', [('raw'), ('cached')])
@pytest.mark.parametrize("filename,index,length,mood,prov_file,prov_line,prov_col", [
# Test results of compositing with the (<) prepend directive
@@ -212,14 +215,15 @@ def load_yaml_file(filename, *, cache_path, shortname=None, from_cache='raw'):
('implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8),
('implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8),
])
-def test_list_composition(datafiles, filename,
+def test_list_composition(datafiles, filename, tmpdir,
index, length, mood,
- prov_file, prov_line, prov_col):
- base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
- overlay = os.path.join(datafiles.dirname, datafiles.basename, filename)
+ prov_file, prov_line, prov_col, caching):
+ base_file = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
+ overlay_file = os.path.join(datafiles.dirname, datafiles.basename, filename)
+
+ base = load_yaml_file(base_file, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
+ overlay = load_yaml_file(overlay_file, cache_path=tmpdir, shortname=filename, from_cache=caching)
- base = _yaml.load(base, shortname='basics.yaml')
- overlay = _yaml.load(overlay, shortname=filename)
_yaml.composite_dict(base, overlay)
children = _yaml.node_get(base, list, 'children')