You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:43:36 UTC
[buildstream] branch jonathan/workspace-fragment-multi-project
created (now b1a8349)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch jonathan/workspace-fragment-multi-project
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at b1a8349 test: start adding tests for multiple elements having the same workspace
This branch includes the following new commits:
new 0506db1 tests: Add a test for multiple projects opening one workspace
new 965da28 Implement basic handling of multiple projects to a single workspace
new 94fe346 WIP: Extend tests to cover repeat opening and closing of workspaces
new b1a8349 test: start adding tests for multiple elements having the same workspace
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 03/04: WIP: Extend tests to cover repeat opening and
closing of workspaces
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jonathan/workspace-fragment-multi-project
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 94fe346e2eab6afa620c5fc04ace3f550454a3f1
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Thu Jan 3 17:04:35 2019 +0000
WIP: Extend tests to cover repeat opening and closing of workspaces
Opening fails, and doing it right is pretty damn complicated.
---
tests/frontend/workspace.py | 101 +++++++++++++++++++++++++++++++++++++-------
1 file changed, 85 insertions(+), 16 deletions(-)
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index b420229..b992538 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -1243,12 +1243,88 @@ def test_external_list(cli, datafiles, tmpdir_factory):
@pytest.mark.datafiles(DATA_DIR)
+def test_multiple_projects_no_force(cli, datafiles, tmpdir_factory):
+ tmpdir1 = tmpdir_factory.mktemp('')
+ tmpdir2 = tmpdir_factory.mktemp('')
+ workspace_dir = os.path.join(str(tmpdir1), "workspace")
+ alpha_project = os.path.join(str(tmpdir1), "alpha-project")
+ beta_project = os.path.join(str(tmpdir2), "beta-project")
+
+ # Open the same workspace with two different projects
+ # without force, we expect this to fail.
+ alpha_element, alpha_project, _ = open_workspace(
+ cli, tmpdir1, datafiles, "git", False, workspace_dir=workspace_dir,
+ project_path=alpha_project, suffix="-alpha"
+ )
+
+ message = "Opening an already-existing workspace without --force should fail"
+ with pytest.raises(AssertionError, message=message):
+ open_workspace(cli, tmpdir2, datafiles, "git", False, workspace_dir=workspace_dir,
+ project_path=beta_project, suffix="-beta", force=False)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_multiple_projects_repeat_open(cli, datafiles, tmpdir_factory):
+ tmpdir1 = tmpdir_factory.mktemp('')
+ tmpdir2 = tmpdir_factory.mktemp('')
+ workspace_dir = os.path.join(str(tmpdir1), "workspace")
+ other_workspace = os.path.join(str(tmpdir1), "workspace2")
+ alpha_project = os.path.join(str(tmpdir1), "alpha-project")
+ beta_project = os.path.join(str(tmpdir2), "beta-project")
+
+ # Open the same workspace with the same project twice to different workspaces
+ # Expect this to succeed because we must use --force for multiple project
+ # behaviour, anyway.
+ # This test mostly exists so that this behaviour stays consistent.
+ alpha_element, alpha_project, _ = open_workspace(
+ cli, tmpdir1, datafiles, "git", False, workspace_dir=workspace_dir,
+ project_path=alpha_project, suffix="-alpha"
+ )
+
+ args = ['-C', workspace_dir, 'workspace', 'open', '-f', '--directory',
+ other_workspace, alpha_element]
+ result = cli.run(project=alpha_project, args=args)
+ result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_multiple_projects_repeat_close(cli, datafiles, tmpdir_factory):
+ tmpdir1 = tmpdir_factory.mktemp('')
+ tmpdir2 = tmpdir_factory.mktemp('')
+ workspace_dir = os.path.join(str(tmpdir1), "workspace")
+ alpha_project = os.path.join(str(tmpdir1), "alpha-project")
+ beta_project = os.path.join(str(tmpdir2), "beta-project")
+
+ alpha_element, alpha_project, _ = open_workspace(
+ cli, tmpdir1, datafiles, "git", False, workspace_dir=workspace_dir,
+ project_path=alpha_project, suffix="-alpha"
+ )
+
+ beta_element, beta_project, _ = open_workspace(
+ cli, tmpdir2, datafiles, "git", False, workspace_dir=workspace_dir,
+ project_path=beta_project, suffix="-beta", force=True
+ )
+
+ # Close the workspace from one project
+ args = ["-C", workspace_dir, "workspace", "close"]
+ result = cli.run(project=alpha_project, args=args)
+ result.assert_success()
+
+ # Close the workspace from the other project
+ result = cli.run(project=beta_project, args=args)
+ result.assert_success()
+
+ # Close the project after it's been closed
+ result = cli.run(project=beta_project, args=args)
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_PROJECT_CONF)
+
+
+@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize(
- "force, close_from_external",
- [(False, False), (True, True), (True, False)],
- ids=["no-force", "close-from-external", "no-close-from-external"]
+ "close_from_external", [(True), (False)],
+ ids=["close-from-external", "no-close-from-external"]
)
-def test_multiple_projects(cli, datafiles, tmpdir_factory, force, close_from_external):
+def test_multiple_projects(cli, datafiles, tmpdir_factory, close_from_external):
# i.e. multiple projects can open the same workspace
tmpdir1 = tmpdir_factory.mktemp('')
tmpdir2 = tmpdir_factory.mktemp('')
@@ -1261,18 +1337,11 @@ def test_multiple_projects(cli, datafiles, tmpdir_factory, force, close_from_ext
cli, tmpdir1, datafiles, "git", False, workspace_dir=workspace_dir,
project_path=alpha_project, suffix="-alpha"
)
- if force:
- beta_element, beta_project, _ = open_workspace(
- cli, tmpdir2, datafiles, "git", False, workspace_dir=workspace_dir,
- project_path=beta_project, suffix="-beta", force=force
- )
- else:
- # Opening a workspace on an existing workspace must only work with "--force"
- message = "Opening an already-existing workspace without --force should fail"
- with pytest.raises(AssertionError, message=message):
- open_workspace(cli, tmpdir2, datafiles, "git", False, workspace_dir=workspace_dir,
- project_path=beta_project, suffix="-beta", force=force)
- return
+
+ beta_element, beta_project, _ = open_workspace(
+ cli, tmpdir2, datafiles, "git", False, workspace_dir=workspace_dir,
+ project_path=beta_project, suffix="-beta", force=True
+ )
# Run a command and assert it came from the alpha-element
# Using element guessing as a way of easily telling which project was used
[buildstream] 01/04: tests: Add a test for multiple projects
opening one workspace
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jonathan/workspace-fragment-multi-project
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0506db1e05584a60ddd8374fcc03cddec7e5a55b
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Tue Dec 11 17:51:24 2018 +0000
tests: Add a test for multiple projects opening one workspace
This change is related to #222
---
tests/frontend/workspace.py | 70 +++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 67 insertions(+), 3 deletions(-)
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index 91004b9..b420229 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -122,7 +122,7 @@ class WorkspaceCreater():
return element_tuples
def open_workspaces(self, kinds, track, suffixs=None, workspace_dir=None,
- element_attrs=None, no_checkout=False):
+ element_attrs=None, no_checkout=False, force=False):
element_tuples = self.create_workspace_elements(kinds, track, suffixs, workspace_dir,
element_attrs)
@@ -135,6 +135,8 @@ class WorkspaceCreater():
args.append('--track')
if no_checkout:
args.append('--no-checkout')
+ if force:
+ args.append('--force')
if workspace_dir is not None:
assert len(element_tuples) == 1, "test logic error"
_, workspace_dir = element_tuples[0]
@@ -161,10 +163,10 @@ class WorkspaceCreater():
def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
- project_path=None, element_attrs=None, no_checkout=False):
+ project_path=None, element_attrs=None, no_checkout=False, force=False):
workspace_object = WorkspaceCreater(cli, tmpdir, datafiles, project_path)
workspaces = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir,
- element_attrs, no_checkout)
+ element_attrs, no_checkout, force)
assert len(workspaces) == 1
element_name, workspace = workspaces[0]
return element_name, workspace_object.project_path, workspace
@@ -1238,3 +1240,65 @@ def test_external_list(cli, datafiles, tmpdir_factory):
result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize(
+ "force, close_from_external",
+ [(False, False), (True, True), (True, False)],
+ ids=["no-force", "close-from-external", "no-close-from-external"]
+)
+def test_multiple_projects(cli, datafiles, tmpdir_factory, force, close_from_external):
+ # i.e. multiple projects can open the same workspace
+ tmpdir1 = tmpdir_factory.mktemp('')
+ tmpdir2 = tmpdir_factory.mktemp('')
+ workspace_dir = os.path.join(str(tmpdir1), "workspace")
+ alpha_project = os.path.join(str(tmpdir1), "alpha-project")
+ beta_project = os.path.join(str(tmpdir2), "beta-project")
+
+ # Open the same workspace with two different projects
+ alpha_element, alpha_project, _ = open_workspace(
+ cli, tmpdir1, datafiles, "git", False, workspace_dir=workspace_dir,
+ project_path=alpha_project, suffix="-alpha"
+ )
+ if force:
+ beta_element, beta_project, _ = open_workspace(
+ cli, tmpdir2, datafiles, "git", False, workspace_dir=workspace_dir,
+ project_path=beta_project, suffix="-beta", force=force
+ )
+ else:
+ # Opening a workspace on an existing workspace must only work with "--force"
+ message = "Opening an already-existing workspace without --force should fail"
+ with pytest.raises(AssertionError, message=message):
+ open_workspace(cli, tmpdir2, datafiles, "git", False, workspace_dir=workspace_dir,
+ project_path=beta_project, suffix="-beta", force=force)
+ return
+
+ # Run a command and assert it came from the alpha-element
+ # Using element guessing as a way of easily telling which project was used
+ result = cli.run(project=alpha_project, args=['-C', workspace_dir, 'show', '--format', '%{name}'])
+ result.assert_success()
+ assert result.output.strip() == alpha_element
+
+ # Close the workspace
+ args = ((["-C", workspace_dir] if close_from_external else []) +
+ ['workspace', 'close'] +
+ ([] if close_from_external else [alpha_element]))
+ result = cli.run(project=alpha_project, args=args)
+ result.assert_success()
+
+ # Check that the 'beta' element is now found
+ result = cli.run(project=beta_project, args=['-C', workspace_dir, 'show', '--format', '%{name}'])
+ result.assert_success()
+ assert result.output.strip() == beta_element
+
+ # Close the workspace again
+ args = ((["-C", workspace_dir] if close_from_external else []) +
+ ['workspace', 'close'] +
+ ([] if close_from_external else [beta_element]))
+ result = cli.run(project=beta_project, args=args)
+ result.assert_success()
+
+ # Check that the workspace no longer works
+ result = cli.run(project=alpha_project, args=['-C', workspace_dir, 'show', '--format', '%{name}'])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_PROJECT_CONF)
[buildstream] 02/04: Implement basic handling of multiple projects
to a single workspace
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jonathan/workspace-fragment-multi-project
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 965da28f342944181b60a3dce72e76cfd153bffe
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Wed Dec 12 15:19:23 2018 +0000
Implement basic handling of multiple projects to a single workspace
This change is related to #222
---
buildstream/_stream.py | 14 ++++--
buildstream/_workspaces.py | 115 +++++++++++++++++++++++++++++++++++----------
2 files changed, 100 insertions(+), 29 deletions(-)
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index caaa489..e107a30 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -656,6 +656,9 @@ class Stream():
self._message(MessageType.INFO, "Creating workspace for element {}"
.format(target.name))
+ # Ensure the WorkspaceProject is loaded before we delete it
+ workspaces.get_workspace_project(directory)
+
workspace = workspaces.get_workspace(target._get_full_name())
if workspace:
workspaces.delete_workspace(target._get_full_name())
@@ -670,7 +673,7 @@ class Stream():
todo_elements = "\nDid not try to create workspaces for " + todo_elements
raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
- workspaces.create_workspace(target, directory, checkout=not no_checkout)
+ workspaces.create_workspace(target, directory, checkout=not no_checkout, append=force)
self._message(MessageType.INFO, "Created a workspace for element: {}"
.format(target._get_full_name()))
@@ -744,6 +747,9 @@ class Stream():
.format(element.name, workspace_path))
continue
+ # Ensure the WorkspaceProject is in the cache before it gets deleted
+ workspaces.get_workspace_project(workspace_path)
+
with element.timed_activity("Removing workspace directory {}"
.format(workspace_path)):
try:
@@ -752,8 +758,10 @@ class Stream():
raise StreamError("Could not remove '{}': {}"
.format(workspace_path, e)) from e
- workspaces.delete_workspace(element._get_full_name())
- workspaces.create_workspace(element, workspace_path, checkout=True)
+ workspaces.delete_workspace(element._get_full_name(),
+ preserve_workspace_project=True)
+ workspaces.create_workspace(element, workspace_path, checkout=True,
+ append=False, preserve_workspace_project=True)
self._message(MessageType.INFO,
"Reset workspace for {} at: {}".format(element.name,
diff --git a/buildstream/_workspaces.py b/buildstream/_workspaces.py
index 24a3cc8..0eac81e 100644
--- a/buildstream/_workspaces.py
+++ b/buildstream/_workspaces.py
@@ -143,8 +143,38 @@ class WorkspaceProject():
# element_name (str): The name of the element that the workspace belongs to.
#
def add_project(self, project_path, element_name):
- assert (project_path and element_name)
- self._projects.append({'project-path': project_path, 'element-name': element_name})
+ # TODO: Decide whether to raise an exception if the project already exists.
+ project = {'project-path': project_path, 'element-name': element_name}
+ if project not in self._projects:
+ self._projects.append(project)
+
+ # set_project()
+ #
+ # Sets the project to only contain the project_path and element_name specified.
+ #
+ # Args:
+ # project_path (str): the path to the project that opened the workspace.
+ # element_name (str): the name of the element that the workspace belongs to.
+ #
+ def set_project(self, project_path, element_name):
+ self._projects = [{'project-path': project_path, 'element-name': element_name}]
+
+ # remove_project()
+ #
+ # Removes the first project entry that matches the project_path and element_name
+ def remove_project(self, project_path, element_name):
+ # NOTE: This will need revisiting if projects' data format changes
+ # TODO: Figure out what to do if there is no project for those parameters
+ self._projects.remove({'project-path': project_path, 'element-name': element_name})
+
+ # has_projects()
+ #
+ # Returns whether there are any projects in this WorkspaceProject
+ #
+ # Returns:
+ # (bool): True if there are any projects, or False if there aren't any
+ def has_projects(self):
+ return any(self._projects)
# WorkspaceProjectCache()
@@ -186,17 +216,21 @@ class WorkspaceProjectCache():
# directory (str): The directory to search for a WorkspaceProject.
# project_path (str): The path to the project that refers to this workspace
# element_name (str): The element in the project that was refers to this workspace
+ # append (bool): Whether the project_path and element_name should be appended
#
# Returns:
# (WorkspaceProject): The WorkspaceProject that was found for that directory.
#
- def add(self, directory, project_path, element_name):
+ def add(self, directory, project_path, element_name, *, append):
workspace_project = self.get(directory)
if not workspace_project:
workspace_project = WorkspaceProject(directory)
self._projects[directory] = workspace_project
- workspace_project.add_project(project_path, element_name)
+ if append:
+ workspace_project.add_project(project_path, element_name)
+ else:
+ workspace_project.set_project(project_path, element_name)
return workspace_project
# remove()
@@ -204,23 +238,28 @@ class WorkspaceProjectCache():
# Removes the project path and element name from the WorkspaceProject that exists
# for that directory.
#
- # NOTE: This currently just deletes the file, but with support for multiple
- # projects opening the same workspace, this will involve decreasing the count
- # and deleting the file if there are no more projects.
- #
# Args:
# directory (str): The directory to search for a WorkspaceProject.
+ # project_path (str): the path to the project that should be removed.
+ # element_name (str): the name of the element in the project that should be removed.
#
- def remove(self, directory):
+ def remove(self, directory, project_path, element_name):
workspace_project = self.get(directory)
if not workspace_project:
raise LoadError(LoadErrorReason.MISSING_FILE,
"Failed to find a {} file to remove".format(WORKSPACE_PROJECT_FILE))
- path = workspace_project.get_filename()
- try:
- os.unlink(path)
- except FileNotFoundError:
- pass
+
+ workspace_project.remove_project(project_path, element_name)
+
+ if workspace_project.has_projects():
+ workspace_project.write()
+ else:
+ # Remove the WorkspaceProject file if it's now empty
+ path = workspace_project.get_filename()
+ try:
+ os.unlink(path)
+ except FileNotFoundError:
+ pass
# Workspace()
@@ -429,8 +468,10 @@ class Workspaces():
# target (Element) - The element to create a workspace for
# path (str) - The path in which the workspace should be kept
# checkout (bool): Whether to check-out the element's sources into the directory
+ # append (bool): Whether the WorkspaceProject file should append this project
+ # preserve_workspace_project (bool): Whether the WorkspaceProject should be altered
#
- def create_workspace(self, target, path, *, checkout):
+ def create_workspace(self, target, path, *, checkout, append, preserve_workspace_project=False):
element_name = target._get_full_name()
project_dir = self._toplevel_project.directory
if path.startswith(project_dir):
@@ -444,7 +485,11 @@ class Workspaces():
with target.timed_activity("Staging sources to {}".format(path)):
target._open_workspace()
- workspace_project = self._workspace_project_cache.add(path, project_dir, element_name)
+ if preserve_workspace_project:
+ workspace_project = self._workspace_project_cache.get(path)
+ else:
+ workspace_project = self._workspace_project_cache.add(path, project_dir, element_name, append=append)
+
project_file_path = workspace_project.get_filename()
if os.path.exists(project_file_path):
@@ -469,6 +514,21 @@ class Workspaces():
return None
return self._workspaces[element_name]
+ # get_workspace_project()
+ #
+ # Returns a WorkspaceProject for a given directory, retrieving from the cache if
+ # present.
+ #
+ # Args:
+ # directory (str): The directory to search for a WorkspaceProject.
+ #
+ # Returns:
+ # (WorkspaceProject): The WorkspaceProject that was found for that directory.
+ # or (NoneType): None, if no WorkspaceProject can be found.
+ #
+ def get_workspace_project(self, directory):
+ return self._workspace_project_cache.get(directory)
+
# update_workspace()
#
# Update the datamodel with a new Workspace instance
@@ -498,20 +558,23 @@ class Workspaces():
#
# Args:
# element_name (str) - The element name whose workspace to delete
+ # preserve_workspace_project (bool): Whether the WorkspaceProject should be altered
#
- def delete_workspace(self, element_name):
+ def delete_workspace(self, element_name, preserve_workspace_project=False):
workspace = self.get_workspace(element_name)
del self._workspaces[element_name]
- # Remove from the cache if it exists
- try:
- self._workspace_project_cache.remove(workspace.get_absolute_path())
- except LoadError as e:
- # We might be closing a workspace with a deleted directory
- if e.reason == LoadErrorReason.MISSING_FILE:
- pass
- else:
- raise
+ if not preserve_workspace_project:
+ # Remove from the cache if it exists
+ project_dir = self._toplevel_project.directory
+ try:
+ self._workspace_project_cache.remove(workspace.get_absolute_path(), project_dir, element_name)
+ except LoadError as e:
+ # We might be closing a workspace with a deleted directory
+ if e.reason == LoadErrorReason.MISSING_FILE:
+ pass
+ else:
+ raise
# save_config()
#
[buildstream] 04/04: test: start adding tests for multiple elements
having the same workspace
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jonathan/workspace-fragment-multi-project
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b1a8349e40c43da20c92d0a9f137b215fb732789
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Thu Jan 3 18:48:31 2019 +0000
test: start adding tests for multiple elements having the same workspace
---
tests/frontend/project/elements/same-repo-bin.bst | 9 +++++++++
tests/frontend/project/elements/same-repo-dev.bst | 9 +++++++++
tests/frontend/workspace.py | 6 ++++++
3 files changed, 24 insertions(+)
diff --git a/tests/frontend/project/elements/same-repo-bin.bst b/tests/frontend/project/elements/same-repo-bin.bst
new file mode 100644
index 0000000..77e1ffc
--- /dev/null
+++ b/tests/frontend/project/elements/same-repo-bin.bst
@@ -0,0 +1,9 @@
+kind: manual
+variables:
+ command-subdir: bin-files
+config:
+ install-commands:
+ - "cp %{build-root}/usr/bin/hello %{install-root}"
+sources:
+- kind: local
+ path: files
diff --git a/tests/frontend/project/elements/same-repo-dev.bst b/tests/frontend/project/elements/same-repo-dev.bst
new file mode 100644
index 0000000..e6b3f87
--- /dev/null
+++ b/tests/frontend/project/elements/same-repo-dev.bst
@@ -0,0 +1,9 @@
+kind: manual
+variables:
+ command-subdir: dev-files
+config:
+ install-commands:
+ - "cp %{build-root}/usr/include/pony.h %{install-root}"
+sources:
+- kind: local
+ path: files
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index b992538..10fd68c 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -1371,3 +1371,9 @@ def test_multiple_projects(cli, datafiles, tmpdir_factory, close_from_external):
# Check that the workspace no longer works
result = cli.run(project=alpha_project, args=['-C', workspace_dir, 'show', '--format', '%{name}'])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_PROJECT_CONF)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_multi_element_multiple_projects(cli, datafiles, tmpdir_factory):
+ # Test that behaviour is consistent when two elements that use the same
+ # repository as a source open the same workspace