You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:22:22 UTC
[buildstream] branch tpollard/752 created (now 51bb134)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch tpollard/752
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 51bb134 Add ability to build without interacting with available artifact servers.
This branch includes the following new commits:
new 99f0d72 _context.py: Add user_remotes to global user context, the default of which is set to 'all'
new 51bb134 Add ability to build without interacting with available artifact servers.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/02: _context.py: Add user_remotes to global user
context, the default of which is set to 'all'
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch tpollard/752
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 99f0d726f993f2c3d5e2a4683fa35d7a28efc2ad
Author: Tom Pollard <to...@codethink.co.uk>
AuthorDate: Wed Nov 14 17:26:34 2018 +0000
_context.py: Add user_remotes to global user context, the default
of which is set to 'all'
_frontend/app.py & cli.py: Add --user-remotes as a bst main option,
with values of 'all', 'user' or 'none' which when passed will
override the default or user defined context for user_remotes
tests/completions/completions.py: Update for the added flag
---
buildstream/_context.py | 13 ++++++++++++-
buildstream/_frontend/app.py | 3 ++-
buildstream/_frontend/cli.py | 3 +++
tests/frontend/completions.py | 2 ++
4 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/buildstream/_context.py b/buildstream/_context.py
index 1d049f7..4375543 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -131,6 +131,9 @@ class Context():
# Make sure the XDG vars are set in the environment before loading anything
self._init_xdg()
+ # Which remote artifact servers to interact with. all, user or none
+ self.use_remotes = 'all'
+
# Private variables
self._cache_key = None
self._message_handler = None
@@ -183,7 +186,8 @@ class Context():
_yaml.node_validate(defaults, [
'sourcedir', 'builddir', 'artifactdir', 'logdir',
'scheduler', 'artifacts', 'logging', 'projects',
- 'cache', 'prompt', 'workspacedir', 'remote-execution'
+ 'cache', 'prompt', 'workspacedir', 'remote-execution',
+ 'useremotes'
])
for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
@@ -213,6 +217,13 @@ class Context():
# Load pull build trees configuration
self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
+ # Load remote artifact server usage
+ self.use_remotes = _yaml.node_get(defaults, str, 'useremotes', default_value='all')
+ valid_actions = ['all', 'user', 'none']
+ if self.use_remotes not in valid_actions:
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "useremotes should be one of: {}".format(", ".join(valid_actions)))
+
# Load logging config
logging = _yaml.node_get(defaults, Mapping, 'logging')
_yaml.node_validate(logging, [
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index b6da079..bd3d475 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -183,7 +183,8 @@ class App():
'builders': 'sched_builders',
'pushers': 'sched_pushers',
'network_retries': 'sched_network_retries',
- 'pull_buildtrees': 'pull_buildtrees'
+ 'pull_buildtrees': 'pull_buildtrees',
+ 'use_remotes': 'use_remotes'
}
for cli_option, context_attr in override_map.items():
option_value = self._main_options.get(cli_option)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 163a057..27f0700 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -252,6 +252,9 @@ def print_version(ctx, param, value):
help="The mirror to fetch from first, before attempting other mirrors")
@click.option('--pull-buildtrees', is_flag=True, default=None,
help="Include an element's build tree when pulling remote element artifacts")
+@click.option('--use-remotes', default='all',
+ type=click.Choice(['all', 'user', 'none']),
+ help='The remote artifact caches to interact with (default: all)')
@click.pass_context
def cli(context, **kwargs):
"""Build and manipulate BuildStream projects
diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py
index 007064f..d7c34d3 100644
--- a/tests/frontend/completions.py
+++ b/tests/frontend/completions.py
@@ -44,6 +44,7 @@ MAIN_OPTIONS = [
"--pull-buildtrees ",
"--pushers ",
"--strict ",
+ "--use-remotes ",
"--verbose ",
"--version ",
]
@@ -155,6 +156,7 @@ def test_options(cli, cmd, word_idx, expected):
@pytest.mark.parametrize("cmd,word_idx,expected", [
('bst --on-error ', 2, ['continue ', 'quit ', 'terminate ']),
+ ('bst --use-remotes ', 2, ['all ', 'user ', 'none ']),
('bst show --deps ', 3, ['all ', 'build ', 'none ', 'plan ', 'run ']),
('bst show --deps=', 2, ['all ', 'build ', 'none ', 'plan ', 'run ']),
('bst show --deps b', 3, ['build ']),
[buildstream] 02/02: Add ability to build without interacting with
available artifact servers.
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch tpollard/752
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 51bb1341e7db35538aca0aa0a1ca273ee373be0c
Author: Tom Pollard <to...@codethink.co.uk>
AuthorDate: Wed Nov 14 17:33:24 2018 +0000
Add ability to build without interacting with available artifact
servers.
By adding a new user context and bst main option, we can provide
more granular control of what remotes are utilised during relevant
bst pull and push actions.
_artifactcache/artifactcache.py: Expand setup_remotes() to consider
the user context value of 'use_remotes'. Allowing the user to omit
interacting with all, or all non user defined remotes.
tests/frontend: pull.py & push.py addition of tests for checking
expected behaviour. Ensures interaction with user, project or no
remotes.
---
NEWS | 9 ++++++
buildstream/_artifactcache.py | 17 ++++++----
tests/frontend/pull.py | 72 +++++++++++++++++++++++++++++++++++++++++++
tests/frontend/push.py | 65 ++++++++++++++++++++++++++++++++++++++
4 files changed, 157 insertions(+), 6 deletions(-)
diff --git a/NEWS b/NEWS
index 449a23f..cf5a22d 100644
--- a/NEWS
+++ b/NEWS
@@ -122,6 +122,15 @@ buildstream 1.3.1
'shell', 'show', 'source-checkout', 'track', 'workspace close' and 'workspace reset'
commands are affected.
+ o bst interaction with defined artifact servers can be controlled more granularly.
+ This can be done via the user configuration option `useremotes` or via the bst cli
+ main option '--use-remotes'. This can be set as 'none', 'user' or the default value
+ 'all'. Unless specifically overriden, when considering wether to pull or push to
+ available artifact servers (be it user or project defined) this optional config option
+ will be used. Setting this value to 'user' for example and performing a build would
+ lead to any project or junction defined artifact server to be ignored, whilst still
+ attempting to any user defined remotes.
+
=================
buildstream 1.1.5
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 48ab278..c162c39 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -148,7 +148,7 @@ class ArtifactCache():
# Sets up which remotes to use
#
# Args:
- # use_config (bool): Whether to use project configuration
+ # use_config (bool): Whether to use configuration
# remote_url (str): Remote artifact cache URL
#
# This requires that all of the projects which are to be processed in the session
@@ -167,11 +167,16 @@ class ArtifactCache():
self._set_remotes([ArtifactCacheSpec(remote_url, push=True)])
has_remote_caches = True
if use_config:
- for project in self.context.get_projects():
- artifact_caches = _configured_remote_artifact_cache_specs(self.context, project)
- if artifact_caches: # artifact_caches is a list of ArtifactCacheSpec instances
- self._set_remotes(artifact_caches, project=project)
- has_remote_caches = True
+ if self.context.use_remotes == 'all':
+ for project in self.context.get_projects():
+ artifact_caches = _configured_remote_artifact_cache_specs(self.context, project)
+ if artifact_caches: # artifact_caches is a list of ArtifactCacheSpec instances
+ self._set_remotes(artifact_caches, project=project)
+ has_remote_caches = True
+ # If configured to only use user configured remotes, pass existing user cache spec
+ elif self.context.use_remotes == 'user' and self.context.artifact_cache_specs:
+ self._set_remotes(self.context.artifact_cache_specs)
+ has_remote_caches = True
if has_remote_caches:
self._initialize_remotes()
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index 20b7409..57e1bfd 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -408,3 +408,75 @@ def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles):
assert "INFO Remote ({}) does not have".format(share.repo) in result.stderr
assert "SKIPPED Pull" in result.stderr
+
+
+# Tests that:
+#
+# * The bst main option --use-remotes limits remote action
+# as expected for pull jobs
+#
+@pytest.mark.datafiles(DATA_DIR)
+def test_useremotes_cli_options(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+
+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject:
+
+ # Add shareproject repo url to project.conf
+ with open(os.path.join(project, "project.conf"), "a") as projconf:
+ projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo))
+
+ # First build the target element and push to the remotes.
+ # We need the artifact available in the remotes to test against.
+ cli.configure({
+ 'artifacts': {'url': shareuser.repo, 'push': True}
+ })
+ result = cli.run(project=project, args=['build', 'target.bst'])
+ result.assert_success()
+ assert cli.get_element_state(project, 'target.bst') == 'cached'
+
+ # Assert that everything is now cached in the remotes.
+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
+ for element_name in all_elements:
+ assert_shared(cli, shareuser, project, element_name)
+ assert_shared(cli, shareproject, project, element_name)
+
+ # Now we've pushed, delete the user's local artifact cache
+ artifacts = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifacts)
+
+ # Assert that nothing is cached locally anymore
+ for element_name in all_elements:
+ assert cli.get_element_state(project, element_name) != 'cached'
+
+ # Attempt bst build with --use-remotes set as none, this should lead to
+ # a complete rebuild without pulling from either artifact remote cache
+ result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst'])
+ result.assert_success()
+ for element_name in all_elements:
+ assert element_name not in result.get_pulled_elements()
+
+ # Delete local cache again
+ artifacts = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifacts)
+
+ # Attempt bst build with --use-remotes set as user, as the shareuser is
+ # passed in as user config and not via a project, assert project remote
+ # was not attempted by it not being in the output
+ result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst'])
+ result.assert_success()
+ for element_name in all_elements:
+ assert element_name in result.get_pulled_elements()
+ assert shareproject.repo not in result.stderr
+
+ # Delete local cache again
+ artifacts = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifacts)
+
+ # Attempt bst build with --use-remotes set as all, this time
+ # assert that project remote is attempted and in the output
+ result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst'])
+ result.assert_success()
+ for element_name in all_elements:
+ assert element_name in result.get_pulled_elements()
+ assert shareproject.repo in result.stderr
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index f3f9b52..b728a2f 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -416,3 +416,68 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles):
assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated"
assert "INFO Remote ({}) already has ".format(share.repo) in result.stderr
assert "SKIPPED Push" in result.stderr
+
+
+# Tests that:
+#
+# * The bst main option --use-remotes limits remote action
+# as expected for push jobs
+#
+@pytest.mark.datafiles(DATA_DIR)
+def test_useremotes_cli_options(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+
+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject:
+
+ # Add shareproject repo url to project.conf
+ with open(os.path.join(project, "project.conf"), "a") as projconf:
+ projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo))
+
+ # Configure shareuser remote in user conf
+ cli.configure({
+ 'artifacts': {'url': shareuser.repo, 'push': True}
+ })
+
+ # First build the target element with --use-remotes set as none.
+ # This should lead to a complete build without pushing to either artifact
+ # remote cache
+ result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst'])
+ result.assert_success()
+ assert not result.get_pushed_elements()
+ assert cli.get_element_state(project, 'target.bst') == 'cached'
+
+ # Delete the artifacts from the local artifact cache
+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
+ for element_name in all_elements:
+ cli.remove_artifact_from_cache(project, element_name)
+
+ # Assert that nothing is cached locally anymore
+ for element_name in all_elements:
+ assert cli.get_element_state(project, element_name) != 'cached'
+
+ # Attempt bst build with --use-remotes set as user, this should lead to
+ # a complete rebuild, with artifacts pushed to the shareuser remote artifact cache
+ # only. Assert project remote was not attempted by it not being in the output
+ result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst'])
+ result.assert_success()
+ for element_name in all_elements:
+ assert element_name in result.get_pushed_elements()
+ for element_name in all_elements:
+ assert_shared(cli, shareuser, project, element_name)
+ assert shareproject.repo not in result.stderr
+
+ # Delete the artifacts from the local artifact cache
+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
+ for element_name in all_elements:
+ cli.remove_artifact_from_cache(project, element_name)
+
+ # Attempt bst build with --use-remotes set as all, this should lead to
+ # a complete rebuild, with artifacts pushed to both the shareuser and
+ # shareproject remote artifacts caches
+ result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst'])
+ result.assert_success()
+ for element_name in all_elements:
+ assert element_name in result.get_pushed_elements()
+ for element_name in all_elements:
+ assert_shared(cli, shareproject, project, element_name)