You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:22:24 UTC
[buildstream] 02/02: Add ability to build without interacting with
available artifact servers.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch tpollard/752
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 51bb1341e7db35538aca0aa0a1ca273ee373be0c
Author: Tom Pollard <to...@codethink.co.uk>
AuthorDate: Wed Nov 14 17:33:24 2018 +0000
Add ability to build without interacting with available artifact
servers.
By adding a new user context and bst main option, we can provide
more granular control of what remotes are utilised during relevant
bst pull and push actions.
_artifactcache/artifactcache.py: Expand setup_remotes() to consider
the user context value of 'use_remotes'. Allowing the user to omit
interacting with all, or all non user defined remotes.
tests/frontend: pull.py & push.py addition of tests for checking
expected behaviour. Ensures interaction with user, project or no
remotes.
---
NEWS | 9 ++++++
buildstream/_artifactcache.py | 17 ++++++----
tests/frontend/pull.py | 72 +++++++++++++++++++++++++++++++++++++++++++
tests/frontend/push.py | 65 ++++++++++++++++++++++++++++++++++++++
4 files changed, 157 insertions(+), 6 deletions(-)
diff --git a/NEWS b/NEWS
index 449a23f..cf5a22d 100644
--- a/NEWS
+++ b/NEWS
@@ -122,6 +122,15 @@ buildstream 1.3.1
'shell', 'show', 'source-checkout', 'track', 'workspace close' and 'workspace reset'
commands are affected.
+ o bst interaction with defined artifact servers can be controlled more granularly.
+ This can be done via the user configuration option `useremotes` or via the bst cli
+ main option '--use-remotes'. This can be set as 'none', 'user' or the default value
+ 'all'. Unless specifically overriden, when considering wether to pull or push to
+ available artifact servers (be it user or project defined) this optional config option
+ will be used. Setting this value to 'user' for example and performing a build would
+ lead to any project or junction defined artifact server to be ignored, whilst still
+ attempting to any user defined remotes.
+
=================
buildstream 1.1.5
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 48ab278..c162c39 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -148,7 +148,7 @@ class ArtifactCache():
# Sets up which remotes to use
#
# Args:
- # use_config (bool): Whether to use project configuration
+ # use_config (bool): Whether to use configuration
# remote_url (str): Remote artifact cache URL
#
# This requires that all of the projects which are to be processed in the session
@@ -167,11 +167,16 @@ class ArtifactCache():
self._set_remotes([ArtifactCacheSpec(remote_url, push=True)])
has_remote_caches = True
if use_config:
- for project in self.context.get_projects():
- artifact_caches = _configured_remote_artifact_cache_specs(self.context, project)
- if artifact_caches: # artifact_caches is a list of ArtifactCacheSpec instances
- self._set_remotes(artifact_caches, project=project)
- has_remote_caches = True
+ if self.context.use_remotes == 'all':
+ for project in self.context.get_projects():
+ artifact_caches = _configured_remote_artifact_cache_specs(self.context, project)
+ if artifact_caches: # artifact_caches is a list of ArtifactCacheSpec instances
+ self._set_remotes(artifact_caches, project=project)
+ has_remote_caches = True
+ # If configured to only use user configured remotes, pass existing user cache spec
+ elif self.context.use_remotes == 'user' and self.context.artifact_cache_specs:
+ self._set_remotes(self.context.artifact_cache_specs)
+ has_remote_caches = True
if has_remote_caches:
self._initialize_remotes()
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index 20b7409..57e1bfd 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -408,3 +408,75 @@ def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles):
assert "INFO Remote ({}) does not have".format(share.repo) in result.stderr
assert "SKIPPED Pull" in result.stderr
+
+
+# Tests that:
+#
+# * The bst main option --use-remotes limits remote action
+# as expected for pull jobs
+#
+@pytest.mark.datafiles(DATA_DIR)
+def test_useremotes_cli_options(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+
+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject:
+
+ # Add shareproject repo url to project.conf
+ with open(os.path.join(project, "project.conf"), "a") as projconf:
+ projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo))
+
+ # First build the target element and push to the remotes.
+ # We need the artifact available in the remotes to test against.
+ cli.configure({
+ 'artifacts': {'url': shareuser.repo, 'push': True}
+ })
+ result = cli.run(project=project, args=['build', 'target.bst'])
+ result.assert_success()
+ assert cli.get_element_state(project, 'target.bst') == 'cached'
+
+ # Assert that everything is now cached in the remotes.
+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
+ for element_name in all_elements:
+ assert_shared(cli, shareuser, project, element_name)
+ assert_shared(cli, shareproject, project, element_name)
+
+ # Now we've pushed, delete the user's local artifact cache
+ artifacts = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifacts)
+
+ # Assert that nothing is cached locally anymore
+ for element_name in all_elements:
+ assert cli.get_element_state(project, element_name) != 'cached'
+
+ # Attempt bst build with --use-remotes set as none, this should lead to
+ # a complete rebuild without pulling from either artifact remote cache
+ result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst'])
+ result.assert_success()
+ for element_name in all_elements:
+ assert element_name not in result.get_pulled_elements()
+
+ # Delete local cache again
+ artifacts = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifacts)
+
+ # Attempt bst build with --use-remotes set as user, as the shareuser is
+ # passed in as user config and not via a project, assert project remote
+ # was not attempted by it not being in the output
+ result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst'])
+ result.assert_success()
+ for element_name in all_elements:
+ assert element_name in result.get_pulled_elements()
+ assert shareproject.repo not in result.stderr
+
+ # Delete local cache again
+ artifacts = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifacts)
+
+ # Attempt bst build with --use-remotes set as all, this time
+ # assert that project remote is attempted and in the output
+ result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst'])
+ result.assert_success()
+ for element_name in all_elements:
+ assert element_name in result.get_pulled_elements()
+ assert shareproject.repo in result.stderr
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index f3f9b52..b728a2f 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -416,3 +416,68 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles):
assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated"
assert "INFO Remote ({}) already has ".format(share.repo) in result.stderr
assert "SKIPPED Push" in result.stderr
+
+
+# Tests that:
+#
+# * The bst main option --use-remotes limits remote action
+# as expected for push jobs
+#
+@pytest.mark.datafiles(DATA_DIR)
+def test_useremotes_cli_options(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+
+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject:
+
+ # Add shareproject repo url to project.conf
+ with open(os.path.join(project, "project.conf"), "a") as projconf:
+ projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo))
+
+ # Configure shareuser remote in user conf
+ cli.configure({
+ 'artifacts': {'url': shareuser.repo, 'push': True}
+ })
+
+ # First build the target element with --use-remotes set as none.
+ # This should lead to a complete build without pushing to either artifact
+ # remote cache
+ result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst'])
+ result.assert_success()
+ assert not result.get_pushed_elements()
+ assert cli.get_element_state(project, 'target.bst') == 'cached'
+
+ # Delete the artifacts from the local artifact cache
+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
+ for element_name in all_elements:
+ cli.remove_artifact_from_cache(project, element_name)
+
+ # Assert that nothing is cached locally anymore
+ for element_name in all_elements:
+ assert cli.get_element_state(project, element_name) != 'cached'
+
+ # Attempt bst build with --use-remotes set as user, this should lead to
+ # a complete rebuild, with artifacts pushed to the shareuser remote artifact cache
+ # only. Assert project remote was not attempted by it not being in the output
+ result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst'])
+ result.assert_success()
+ for element_name in all_elements:
+ assert element_name in result.get_pushed_elements()
+ for element_name in all_elements:
+ assert_shared(cli, shareuser, project, element_name)
+ assert shareproject.repo not in result.stderr
+
+ # Delete the artifacts from the local artifact cache
+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
+ for element_name in all_elements:
+ cli.remove_artifact_from_cache(project, element_name)
+
+ # Attempt bst build with --use-remotes set as all, this should lead to
+ # a complete rebuild, with artifacts pushed to both the shareuser and
+ # shareproject remote artifacts caches
+ result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst'])
+ result.assert_success()
+ for element_name in all_elements:
+ assert element_name in result.get_pushed_elements()
+ for element_name in all_elements:
+ assert_shared(cli, shareproject, project, element_name)