You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ro...@apache.org on 2020/12/29 13:47:45 UTC
[buildstream] 01/01: WIP: Opening a workspace with a cached build
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tpollard/workspacebuildtree
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 05fc1bf443a91cd2515dcfb4df80864fa8060e66
Author: Tom Pollard <to...@codethink.co.uk>
AuthorDate: Mon Oct 1 10:56:09 2018 +0100
WIP: Opening a workspace with a cached build
---
buildstream/_artifactcache.py | 14 ++++
buildstream/_cas/cascache.py | 15 ++++
buildstream/_context.py | 9 ++-
buildstream/_frontend/cli.py | 17 +++-
buildstream/_gitsourcebase.py | 33 ++++++++
buildstream/_stream.py | 62 +++++++++++++--
buildstream/_workspaces.py | 17 ++--
buildstream/element.py | 17 +++-
buildstream/source.py | 24 ++++++
doc/source/developing/workspaces.rst | 32 ++++++++
tests/frontend/workspace.py | 146 ++++++++++++++++++++++++++++++-----
tests/integration/workspace.py | 36 +++++++++
12 files changed, 385 insertions(+), 37 deletions(-)
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 5404dc1..66d54eb 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -855,6 +855,20 @@ class ArtifactCache():
self.cas.link_ref(oldref, newref)
+ # checkout_artifact_subdir()
+ #
+ # Checkout given artifact subdir into provided directory
+ #
+ # Args:
+ # element (Element): The Element
+ # key (str): The cache key to use
+ # subdir (str): The subdir to checkout
+ # tmpdir (str): The dir to place the subdir content
+ #
+ def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
+ ref = self.get_artifact_fullname(element, key)
+ return self.cas.checkout_artifact_subdir(ref, subdir, tmpdir)
+
################################################
# Local Private Methods #
################################################
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 9d7a121..6d37c1c 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -589,6 +589,21 @@ class CASCache():
reachable = set()
self._reachable_refs_dir(reachable, tree, update_mtime=True)
+ # checkout_artifact_subdir():
+ #
+ # Checkout given artifact subdir into provided directory
+ #
+ # Args:
+ # ref (str): The ref to check
+ # subdir (str): The subdir to checkout
+ # tmpdir (str): The dir to place the subdir content
+ #
+ def checkout_artifact_subdir(self, ref, subdir, tmpdir):
+ tree = self.resolve_ref(ref)
+ # This assumes that the subdir digest is present in the element tree
+ subdirdigest = self._get_subdir(tree, subdir)
+ self._checkout(tmpdir, subdirdigest)
+
################################################
# Local Private Methods #
################################################
diff --git a/buildstream/_context.py b/buildstream/_context.py
index 1d049f7..dee260d 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -125,6 +125,9 @@ class Context():
# close the workspace when they're using it to access the project.
self.prompt_workspace_close_project_inaccessible = None
+ # Whether to include artifact buildtrees in workspaces if available
+ self.workspace_buildtrees = True
+
# Whether elements must be rebuilt when their dependencies have changed
self._strict_build_plan = None
@@ -183,7 +186,8 @@ class Context():
_yaml.node_validate(defaults, [
'sourcedir', 'builddir', 'artifactdir', 'logdir',
'scheduler', 'artifacts', 'logging', 'projects',
- 'cache', 'prompt', 'workspacedir', 'remote-execution'
+ 'cache', 'prompt', 'workspacedir', 'remote-execution',
+ 'workspace-buildtrees'
])
for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
@@ -213,6 +217,9 @@ class Context():
# Load pull build trees configuration
self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
+ # Load workspace buildtrees configuration
+ self.workspace_buildtrees = _yaml.node_get(defaults, bool, 'workspace-buildtrees', default_value='True')
+
# Load logging config
logging = _yaml.node_get(defaults, Mapping, 'logging')
_yaml.node_validate(logging, [
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 49d5717..5437acb 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -773,7 +773,7 @@ def workspace():
##################################################################
@workspace.command(name='open', short_help="Open a new workspace")
@click.option('--no-checkout', default=False, is_flag=True,
- help="Do not checkout the source, only link to the given directory")
+ help="Do not checkout the source or cached buildtree, only link to the given directory")
@click.option('--force', '-f', default=False, is_flag=True,
help="The workspace will be created even if the directory in which it will be created is not empty " +
"or if a workspace for that element already exists")
@@ -782,16 +782,25 @@ def workspace():
@click.option('--directory', type=click.Path(file_okay=False), default=None,
help="Only for use when a single Element is given: Set the directory to use to create the workspace")
@click.argument('elements', nargs=-1, type=click.Path(readable=False), required=True)
+@click.option('--no-cache', default=False, is_flag=True,
+ help="Do not checkout the cached buildtree")
@click.pass_obj
-def workspace_open(app, no_checkout, force, track_, directory, elements):
- """Open a workspace for manual source modification"""
+def workspace_open(app, no_checkout, force, track_, directory, elements, no_cache):
+
+ """Open a workspace for manual source modification, the elements buildtree
+ will be provided if available in the local artifact cache.
+ """
+
+ if not no_cache and not no_checkout:
+ click.echo("WARNING: Workspace will be opened without the cached buildtree if not cached locally")
with app.initialized():
app.stream.workspace_open(elements,
no_checkout=no_checkout,
track_first=track_,
force=force,
- custom_dir=directory)
+ custom_dir=directory,
+ no_cache=no_cache)
##################################################################
diff --git a/buildstream/_gitsourcebase.py b/buildstream/_gitsourcebase.py
index b986f75..b71864c 100644
--- a/buildstream/_gitsourcebase.py
+++ b/buildstream/_gitsourcebase.py
@@ -223,6 +223,31 @@ class GitMirror(SourceFetcher):
fail="Failed to checkout git ref {}".format(self.ref),
cwd=fullpath)
+ def init_cached_build_workspace(self, directory):
+ fullpath = os.path.join(directory, self.path)
+ url = self.source.translate_url(self.url)
+
+ self.source.call([self.source.host_git, 'init', fullpath],
+ fail="Failed to init git in directory: {}".format(fullpath),
+ fail_temporarily=True,
+ cwd=fullpath)
+
+ self.source.call([self.source.host_git, 'fetch', self.mirror],
+ fail='Failed to fetch from local mirror "{}"'.format(self.mirror),
+ cwd=fullpath)
+
+ self.source.call([self.source.host_git, 'remote', 'add', 'origin', url],
+ fail='Failed to add remote origin "{}"'.format(url),
+ cwd=fullpath)
+
+ self.source.call([self.source.host_git, 'update-ref', '--no-deref', 'HEAD', self.ref],
+ fail='Failed update HEAD to ref "{}"'.format(self.ref),
+ cwd=fullpath)
+
+ self.source.call([self.source.host_git, 'read-tree', 'HEAD'],
+ fail='Failed to read HEAD into index',
+ cwd=fullpath)
+
# List the submodules (path/url tuples) present at the given ref of this repo
def submodule_list(self):
modules = "{}:{}".format(self.ref, GIT_MODULES)
@@ -522,6 +547,14 @@ class _GitSourceBase(Source):
for mirror in self.submodules:
mirror.init_workspace(directory)
+ def init_cached_build_workspace(self, directory):
+ self._refresh_submodules()
+
+ with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
+ self.mirror.init_cached_build_workspace(directory)
+ for mirror in self.submodules:
+ mirror.init_cached_build_workspace(directory)
+
def stage(self, directory):
# Need to refresh submodule list here again, because
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index 5887805..d5ac505 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -541,14 +541,21 @@ class Stream():
# track_first (bool): Whether to track and fetch first
# force (bool): Whether to ignore contents in an existing directory
# custom_dir (str): Custom location to create a workspace or false to use default location.
+ # no_cache (bool): Whether to not include the cached buildtree
#
def workspace_open(self, targets, *,
no_checkout,
track_first,
force,
- custom_dir):
+ custom_dir,
+ no_cache):
+
# This function is a little funny but it is trying to be as atomic as possible.
+ # Set no_cache if the global user conf workspacebuildtrees is false
+ if not self._context.workspace_buildtrees:
+ no_cache = True
+
if track_first:
track_targets = targets
else:
@@ -606,7 +613,7 @@ class Stream():
directory = os.path.abspath(custom_dir)
expanded_directories = [directory, ]
else:
- # If this fails it is a bug in what ever calls this, usually cli.py and so can not be tested for via the
+ # If this fails it is a bug in whatever calls this, usually cli.py and so can not be tested for via the
# run bst test mechanism.
assert len(elements) == len(expanded_directories)
@@ -621,12 +628,26 @@ class Stream():
.format(target.name, directory), reason='bad-directory')
# So far this function has tried to catch as many issues as possible with out making any changes
- # Now it dose the bits that can not be made atomic.
+ # Now it does the bits that can not be made atomic.
targetGenerator = zip(elements, expanded_directories)
for target, directory in targetGenerator:
self._message(MessageType.INFO, "Creating workspace for element {}"
.format(target.name))
+ # Check if given target has a buildtree artifact cached locally
+ buildtree = None
+ if target._cached():
+ buildtree = target._cached_buildtree()
+
+ # If we're running in the default state, make the user aware of buildtree usage
+ if not no_cache and not no_checkout:
+ if buildtree:
+ self._message(MessageType.INFO, "{} buildtree artifact is available,"
+ " workspace will be opened with it".format(target.name))
+ else:
+ self._message(MessageType.WARN, "{} buildtree artifact not available,"
+ " workspace will be opened with source checkout".format(target.name))
+
workspace = workspaces.get_workspace(target._get_full_name())
if workspace:
workspaces.delete_workspace(target._get_full_name())
@@ -641,7 +662,20 @@ class Stream():
todo_elements = "\nDid not try to create workspaces for " + todo_elements
raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
- workspaces.create_workspace(target, directory, checkout=not no_checkout)
+ # Handle opening workspace with buildtree included
+ if (buildtree and not no_cache) and not no_checkout:
+ workspaces.create_workspace(target, directory, checkout=not no_checkout, cached_build=buildtree)
+ with target.timed_activity("Staging buildtree to {}".format(directory)):
+ target._open_workspace(buildtree=buildtree)
+ else:
+ workspaces.create_workspace(target, directory, checkout=not no_checkout)
+ if (not buildtree or no_cache) and not no_checkout:
+ with target.timed_activity("Staging sources to {}".format(directory)):
+ target._open_workspace()
+
+ # Saving the workspace once it is set up means that if the next workspace fails to be created before
+ # the configuration gets saved. The successfully created workspace still gets saved.
+ workspaces.save_config()
self._message(MessageType.INFO, "Created a workspace for element: {}"
.format(target._get_full_name()))
@@ -724,7 +758,25 @@ class Stream():
.format(workspace_path, e)) from e
workspaces.delete_workspace(element._get_full_name())
- workspaces.create_workspace(element, workspace_path, checkout=True)
+
+ # Create the workspace, ensuring the original optional cached build state is preserved if
+ # possible.
+ buildtree = False
+ if workspace.cached_build and element._cached():
+ if self._artifacts.contains_subdir_artifact(element, element._get_cache_key(), 'buildtree'):
+ buildtree = True
+
+ # Warn the user if the workspace cannot be opened with the original cached build state
+ if workspace.cached_build and not buildtree:
+ self._message(MessageType.WARN, "{} original buildtree artifact not available,"
+ " workspace will be opened with source checkout".format(element.name))
+
+ # If opening the cached build, set checkout to false
+ workspaces.create_workspace(element, workspace_path,
+ checkout=not buildtree, cached_build=buildtree)
+
+ with element.timed_activity("Staging to {}".format(workspace_path)):
+ element._open_workspace(buildtree=buildtree)
self._message(MessageType.INFO,
"Reset workspace for {} at: {}".format(element.name,
diff --git a/buildstream/_workspaces.py b/buildstream/_workspaces.py
index 24a3cc8..30e30e7 100644
--- a/buildstream/_workspaces.py
+++ b/buildstream/_workspaces.py
@@ -24,7 +24,7 @@ from . import _yaml
from ._exceptions import LoadError, LoadErrorReason
-BST_WORKSPACE_FORMAT_VERSION = 3
+BST_WORKSPACE_FORMAT_VERSION = 4
BST_WORKSPACE_PROJECT_FORMAT_VERSION = 1
WORKSPACE_PROJECT_FILE = ".bstproject.yaml"
@@ -239,9 +239,11 @@ class WorkspaceProjectCache():
# running_files (dict): A dict mapping dependency elements to files
# changed between failed builds. Should be
# made obsolete with failed build artifacts.
+# cached_build (bool): If the workspace is staging the cached build artifact
#
class Workspace():
- def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
+ def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False,
+ running_files=None, cached_build=False):
self.prepared = prepared
self.last_successful = last_successful
self._path = path
@@ -249,6 +251,7 @@ class Workspace():
self._toplevel_project = toplevel_project
self._key = None
+ self.cached_build = cached_build
# to_dict()
#
@@ -261,7 +264,8 @@ class Workspace():
ret = {
'prepared': self.prepared,
'path': self._path,
- 'running_files': self.running_files
+ 'running_files': self.running_files,
+ 'cached_build': self.cached_build
}
if self.last_successful is not None:
ret["last_successful"] = self.last_successful
@@ -429,8 +433,9 @@ class Workspaces():
# target (Element) - The element to create a workspace for
# path (str) - The path in which the workspace should be kept
# checkout (bool): Whether to check-out the element's sources into the directory
+ # cached_build (bool) - If the workspace is staging the cached build artifact
#
- def create_workspace(self, target, path, *, checkout):
+ def create_workspace(self, target, path, *, checkout, cached_build=False):
element_name = target._get_full_name()
project_dir = self._toplevel_project.directory
if path.startswith(project_dir):
@@ -438,7 +443,8 @@ class Workspaces():
else:
workspace_path = path
- self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path)
+ self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path,
+ cached_build=cached_build)
if checkout:
with target.timed_activity("Staging sources to {}".format(path)):
@@ -627,6 +633,7 @@ class Workspaces():
'path': _yaml.node_get(node, str, 'path'),
'last_successful': _yaml.node_get(node, str, 'last_successful', default_value=None),
'running_files': _yaml.node_get(node, dict, 'running_files', default_value=None),
+ 'cached_build': _yaml.node_get(node, bool, 'cached_build', default_value=False)
}
return Workspace.from_dict(self._toplevel_project, dictionary)
diff --git a/buildstream/element.py b/buildstream/element.py
index a243826..59af874 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1909,7 +1909,10 @@ class Element(Plugin):
# This requires that a workspace already be created in
# the workspaces metadata first.
#
- def _open_workspace(self):
+ # Args:
+ # buildtree (bool): Whether to open workspace with artifact buildtree
+ #
+ def _open_workspace(self, buildtree=False):
context = self._get_context()
workspace = self._get_workspace()
assert workspace is not None
@@ -1922,11 +1925,19 @@ class Element(Plugin):
# files in the target directory actually works without any
# additional support from Source implementations.
#
+
os.makedirs(context.builddir, exist_ok=True)
with utils._tempdir(dir=context.builddir, prefix='workspace-{}'
.format(self.normal_name)) as temp:
- for source in self.sources():
- source._init_workspace(temp)
+
+ # Checkout cached buildtree, augment with source plugin if applicable
+ if buildtree:
+ self.__artifacts.checkout_artifact_subdir(self, self._get_cache_key(), 'buildtree', temp)
+ for source in self.sources():
+ source._init_cached_build_workspace(temp)
+ else:
+ for source in self.sources():
+ source._init_workspace(temp)
# Now hardlink the files into the workspace target.
utils.link_files(temp, workspace.get_absolute_path())
diff --git a/buildstream/source.py b/buildstream/source.py
index 9e9bad7..4a7fd0d 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -465,6 +465,24 @@ class Source(Plugin):
"""
self.stage(directory)
+ def init_cached_build_workspace(self, directory):
+ """Initialises a new cached build workspace
+
+ Args:
+ directory (str): Path of the workspace to init
+
+ Raises:
+ :class:`.SourceError`
+
+ Implementors overriding this method should assume that *directory*
+ already exists.
+
+ Implementors should raise :class:`.SourceError` when encountering
+ some system error.
+ """
+ # Allow a non implementation
+ return None
+
def get_source_fetchers(self):
"""Get the objects that are used for fetching
@@ -717,6 +735,12 @@ class Source(Plugin):
self.init_workspace(directory)
+ # Wrapper for init_cached_build_workspace()
+ def _init_cached_build_workspace(self, directory):
+ directory = self.__ensure_directory(directory)
+
+ self.init_cached_build_workspace(directory)
+
# _get_unique_key():
#
# Wrapper for get_unique_key() api
diff --git a/doc/source/developing/workspaces.rst b/doc/source/developing/workspaces.rst
index 653716f..0564bec 100644
--- a/doc/source/developing/workspaces.rst
+++ b/doc/source/developing/workspaces.rst
@@ -24,9 +24,32 @@ Suppose we now want to alter the functionality of the *hello* command. We can
make changes to the source code of Buildstream elements by making use of
BuildStream's workspace command.
+Utilising cached buildtrees
+---------------------------
+ When a BuildStream build element artifact is created and cached, a snapshot of
+ the build directory after the build commands have completed is included in the
+ artifact. This `build tree` can be considered an intermediary state of element,
+ where the source is present along with any output created during the build
+ execution.
+
+ By default when opening a workspace, bst will attempt to stage the build tree
+ into the workspace if it's available in the local cache. If the respective
+ build tree is not present in the cache (element not cached, partially cached or
+ is a non build element) then the source will be staged as is. The default
+ behaviour to attempt to use the build tree can be overriden with specific bst
+ workspace open option of `--no-cache`, or via setting user configuration option
+ `workspacebuildtrees: False`
+
Opening a workspace
-------------------
+.. note::
+
+ This example presumes you built the hello.bst during
+ :ref:`running commands <tutorial_running_commands>`
+ if not, please start by building it.
+
+
First we need to open a workspace, we can do this by running
.. raw:: html
@@ -93,6 +116,15 @@ Alternatively, if we wish to discard the changes we can use
This resets the workspace to its original state.
+.. note::
+
+ bst reset will attempt to open the workspace in
+ the condition in which it was originally staged,
+ i.e with or without consuming the element build tree.
+ If it was originally staged with a cached build tree
+ and there's no longer one available, the source will
+ be staged as is.
+
To discard the workspace completely we can do:
.. raw:: html
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index f6d12e8..31eab76 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -93,18 +93,18 @@ class WorkspaceCreater():
element_name))
return element_name, element_path, workspace_dir
- def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None,
+ def create_workspace_elements(self, kinds, track, suffixes=None, workspace_dir_usr=None,
element_attrs=None):
element_tuples = []
- if suffixs is None:
- suffixs = ['', ] * len(kinds)
+ if suffixes is None:
+ suffixes = ['', ] * len(kinds)
else:
- if len(suffixs) != len(kinds):
+ if len(suffixes) != len(kinds):
raise "terable error"
- for suffix, kind in zip(suffixs, kinds):
+ for suffix, kind in zip(suffixes, kinds):
element_name, element_path, workspace_dir = \
self.create_workspace_element(kind, track, suffix, workspace_dir_usr,
element_attrs)
@@ -121,10 +121,10 @@ class WorkspaceCreater():
return element_tuples
- def open_workspaces(self, kinds, track, suffixs=None, workspace_dir=None,
- element_attrs=None, no_checkout=False):
+ def open_workspaces(self, kinds, track, suffixes=None, workspace_dir=None,
+ element_attrs=None, no_checkout=False, no_cache=False):
- element_tuples = self.create_workspace_elements(kinds, track, suffixs, workspace_dir,
+ element_tuples = self.create_workspace_elements(kinds, track, suffixes, workspace_dir,
element_attrs)
os.makedirs(self.workspace_cmd, exist_ok=True)
@@ -135,12 +135,15 @@ class WorkspaceCreater():
args.append('--track')
if no_checkout:
args.append('--no-checkout')
+ if no_cache:
+ args.append('--no-cache')
if workspace_dir is not None:
assert len(element_tuples) == 1, "test logic error"
_, workspace_dir = element_tuples[0]
args.extend(['--directory', workspace_dir])
-
+ print("element_tuples", element_tuples)
args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
+ print("args", args)
result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args)
result.assert_success()
@@ -157,14 +160,14 @@ class WorkspaceCreater():
filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
assert os.path.exists(filename)
- return element_tuples
+ return element_tuples, result
def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
- project_path=None, element_attrs=None, no_checkout=False):
+ project_path=None, element_attrs=None, no_checkout=False, no_cache=False):
workspace_object = WorkspaceCreater(cli, tmpdir, datafiles, project_path)
- workspaces = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir,
- element_attrs, no_checkout)
+ workspaces, _ = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir,
+ element_attrs, no_checkout, no_cache)
assert len(workspaces) == 1
element_name, workspace = workspaces[0]
return element_name, workspace_object.project_path, workspace
@@ -198,7 +201,7 @@ def test_open_bzr_customize(cli, tmpdir, datafiles):
def test_open_multi(cli, tmpdir, datafiles):
workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
- workspaces = workspace_object.open_workspaces(repo_kinds, False)
+ workspaces, _ = workspace_object.open_workspaces(repo_kinds, False)
for (elname, workspace), kind in zip(workspaces, repo_kinds):
assert kind in elname
@@ -832,7 +835,9 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
"alpha.bst": {
"prepared": False,
"path": "/workspaces/bravo",
- "running_files": {}
+ "running_files": {},
+ "cached_build": False
+
}
}
}),
@@ -847,7 +852,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
"alpha.bst": {
"prepared": False,
"path": "/workspaces/bravo",
- "running_files": {}
+ "running_files": {},
+ "cached_build": False
}
}
}),
@@ -865,7 +871,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
"alpha.bst": {
"prepared": False,
"path": "/workspaces/bravo",
- "running_files": {}
+ "running_files": {},
+ "cached_build": False
}
}
}),
@@ -890,7 +897,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
"last_successful": "some_key",
"running_files": {
"beta.bst": ["some_file"]
- }
+ },
+ "cached_build": False
}
}
}),
@@ -910,7 +918,30 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
"alpha.bst": {
"prepared": True,
"path": "/workspaces/bravo",
- "running_files": {}
+ "running_files": {},
+ "cached_build": False
+ }
+ }
+ }),
+ # Test loading version 4
+ ({
+ "format-version": 4,
+ "workspaces": {
+ "alpha.bst": {
+ "prepared": False,
+ "path": "/workspaces/bravo",
+ "running_files": {},
+ "cached_build": True
+ }
+ }
+ }, {
+ "format-version": BST_WORKSPACE_FORMAT_VERSION,
+ "workspaces": {
+ "alpha.bst": {
+ "prepared": False,
+ "path": "/workspaces/bravo",
+ "running_files": {},
+ "cached_build": True
}
}
})
@@ -1236,3 +1267,80 @@ def test_external_list(cli, datafiles, tmpdir_factory):
result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_nocache_open_messages(cli, tmpdir, datafiles):
+
+ workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
+ _, result = workspace_object.open_workspaces(('git', ), False)
+
+ # cli default WARN for source dropback possibility when no-cache flag is not passed
+ assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" in result.output
+
+ # cli WARN for source dropback happening when no-cache flag not given, but buildtree not available
+ assert "workspace will be opened with source checkout" in result.stderr
+
+ # cli default WARN for source dropback possibilty not given when no-cache flag is passed
+ tmpdir = os.path.join(str(tmpdir), "2")
+ workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
+ _, result = workspace_object.open_workspaces(('git', ), False, suffixes='1', no_cache=True)
+
+ assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" not in result.output
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_nocache_reset_messages(cli, tmpdir, datafiles):
+
+ workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
+ workspaces, result = workspace_object.open_workspaces(('git', ), False)
+ element_name, workspace = workspaces[0]
+ project = workspace_object.project_path
+
+ # Modify workspace, without building so the artifact is not cached
+ shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
+ os.makedirs(os.path.join(workspace, 'etc'))
+ with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
+ f.write("PONY='pink'")
+
+ # Now reset the open workspace, this should have the
+ # effect of reverting our changes to the original source, as it
+ # was not originally opened with a cached buildtree and as such
+ # should not notify the user
+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
+ 'workspace', 'reset', element_name
+ ])
+ result.assert_success()
+ assert "original buildtree artifact not available" not in result.output
+ assert os.path.exists(os.path.join(workspace, 'usr', 'bin', 'hello'))
+ assert not os.path.exists(os.path.join(workspace, 'etc', 'pony.conf'))
+
+ # Close the workspace
+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
+ 'workspace', 'close', '--remove-dir', element_name
+ ])
+ result.assert_success()
+
+ # Build the workspace so we have a cached buildtree artifact for the element
+ assert cli.get_element_state(project, element_name) == 'buildable'
+ result = cli.run(project=project, args=['build', element_name])
+ result.assert_success()
+
+ # Opening the workspace after a build should lead to the cached buildtree being
+ # staged by default
+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
+ 'workspace', 'open', element_name
+ ])
+ result.assert_success()
+
+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
+ 'workspace', 'list'
+ ])
+ result.assert_success()
+ # Now reset the workspace and ensure that a warning is not given about the artifact
+ # buildtree not being available
+ result = cli.run(cwd=workspace_object.workspace_cmd, project=project, args=[
+ 'workspace', 'reset', element_name
+ ])
+ result.assert_success()
+ assert "original buildtree artifact not available" not in result.output
diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py
index f1d8d6f..08bb7ad 100644
--- a/tests/integration/workspace.py
+++ b/tests/integration/workspace.py
@@ -278,3 +278,39 @@ def test_incremental_configure_commands_run_only_once(cli, tmpdir, datafiles):
res = cli.run(project=project, args=['build', element_name])
res.assert_success()
assert not os.path.exists(os.path.join(workspace, 'prepared-again'))
+
+
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+def test_workspace_contains_buildtree(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ workspace = os.path.join(cli.directory, 'workspace')
+ element_name = 'autotools/amhello.bst'
+
+ # Ensure we're not using the shared artifact cache
+ cli.configure({
+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
+ })
+
+ # First open the workspace
+ res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res.assert_success()
+
+ # Check that by default the buildtree wasn't staged as not yet available in the cache
+ assert not os.path.exists(os.path.join(workspace, 'src', 'hello'))
+
+ # Close the workspace, removing the dir
+ res = cli.run(project=project, args=['workspace', 'close', '--remove-dir', element_name])
+ res.assert_success()
+
+ # Build the element, so we have it cached along with the buildtreee
+ res = cli.run(project=project, args=['build', element_name])
+ res.assert_success()
+
+ # Open up the workspace, as the buildtree is cached by default it should open with the buildtree
+ res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res.assert_success()
+
+ # Check that the buildtree was staged, by asserting output of the build exists in the dir
+ assert os.path.exists(os.path.join(workspace, 'src', 'hello'))