You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ro...@apache.org on 2020/12/29 13:45:33 UTC
[buildstream] branch tristan/multiple-caches created (now 0bd2e3b)
This is an automated email from the ASF dual-hosted git repository.
root pushed a change to branch tristan/multiple-caches
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 0bd2e3b _artifactcache/ostreecache.py: Handle ^C and shutdown child task when initializing cache
This branch includes the following new commits:
new f4c007e tests/frontend: Assert cached state in a single line
new f344744 Add support for multiple remote caches
new 536c7e0 Add `bst push --remote` and `bst pull --remote`
new 518ec59 tests: Exercise the new multiple cache support
new 709973f doc: Document multiple cache support
new 0bd2e3b _artifactcache/ostreecache.py: Handle ^C and shutdown child task when initializing cache
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 04/06: tests: Exercise the new multiple cache support
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tristan/multiple-caches
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 518ec59f73125a52c6c689dbc016f99d25c4e71d
Author: Sam Thursfield <sa...@codethink.co.uk>
AuthorDate: Tue Nov 28 12:03:10 2017 +0000
tests: Exercise the new multiple cache support
---
tests/frontend/pull.py | 132 ++++++++++++++++++++++++++++++++-------
tests/frontend/push.py | 23 +++----
tests/testutils/artifactshare.py | 41 ++++++++----
tests/testutils/runcli.py | 19 ++++++
4 files changed, 171 insertions(+), 44 deletions(-)
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index 18a4b46..6920566 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -4,8 +4,6 @@ import pytest
from tests.testutils import cli, create_artifact_share, configure_remote_caches
from tests.testutils.site import IS_LINUX
-from buildstream import _yaml
-
# Project directory
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
@@ -25,16 +23,30 @@ def assert_shared(cli, share, project, element_name):
.format(share.repo, element_name))
+# Assert that a given artifact is NOT in the share
+#
+def assert_not_shared(cli, share, project, element_name):
+ # NOTE: 'test' here is the name of the project
+ # specified in the project.conf we are testing with.
+ #
+ cache_key = cli.get_element_key(project, element_name)
+ if share.has_artifact('test', element_name, cache_key):
+ raise AssertionError("Artifact share at {} unexpectedly contains the element {}"
+ .format(share.repo, element_name))
+
+
@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
@pytest.mark.parametrize(
- 'override_url, project_url, user_url',
+ 'override_urls, project_urls, user_urls',
[
- pytest.param(None, None, 'share.repo', id='user-config'),
- pytest.param(None, 'share.repo', None, id='project-config'),
- pytest.param('share.repo', None, None, id='project-override-in-user-config'),
+ # The leftmost cache is the highest priority one.
+ pytest.param([], [], ['share.repo', '/tmp/do-not-use/user'], id='user-config'),
+ pytest.param([], ['share.repo', '/tmp/do-not-use/project'], ['/tmp/do-not-use/user'], id='project-config'),
+ pytest.param(['share.repo'], ['/tmp/do-not-use/project'], ['/tmp/do-not-use/user'],
+ id='project-override-in-user-config'),
])
@pytest.mark.datafiles(DATA_DIR)
-def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
+def test_push_pull(cli, tmpdir, datafiles, override_urls, project_urls, user_urls):
project = os.path.join(datafiles.dirname, datafiles.basename)
share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'))
@@ -45,25 +57,23 @@ def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
# Assert that we are now cached locally
assert cli.get_element_state(project, 'import-bin.bst') == 'cached'
- override_url = share.repo if override_url == 'share.repo' else override_url
- project_url = share.repo if project_url == 'share.repo' else project_url
- user_url = share.repo if user_url == 'share.repo' else user_url
+ override_urls = [share.repo if url == 'share.repo' else url for url in override_urls]
+ project_urls = [share.repo if url == 'share.repo' else url for url in project_urls]
+ user_urls = [share.repo if url == 'share.repo' else url for url in user_urls]
+ # Configure artifact share
project_conf_file = str(datafiles.join('project.conf'))
- configure_remote_caches(cli, project_conf_file, override_url, project_url, user_url)
+ configure_remote_caches(cli, project_conf_file, override_urls, project_urls, user_urls)
+ share.update_summary()
- # Now try bst push
+ # Now try bst push. This will push to the highest priority cache.
result = cli.run(project=project, args=['push', 'import-bin.bst'])
result.assert_success()
+ share.update_summary()
# And finally assert that the artifact is in the share
assert_shared(cli, share, project, 'import-bin.bst')
- # Make sure we update the summary in our artifact share,
- # we dont have a real server around to do it
- #
- share.update_summary()
-
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
@@ -116,17 +126,13 @@ def test_push_pull_all(cli, tmpdir, datafiles):
# Now try bst push
result = cli.run(project=project, args=['push', '--deps', 'all', 'target.bst'])
result.assert_success()
+ share.update_summary()
# And finally assert that the artifact is in the share
all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst']
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
- # Make sure we update the summary in our artifact share,
- # we dont have a real server around to do it
- #
- share.update_summary()
-
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
@@ -145,3 +151,85 @@ def test_push_pull_all(cli, tmpdir, datafiles):
# And assert that it's again in the local cache, without having built
for element_name in all_elements:
assert cli.get_element_state(project, element_name) == 'cached'
+
+
+@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
+@pytest.mark.datafiles(DATA_DIR)
+def test_push_pull_specific_remote(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+
+ good_share = create_artifact_share(os.path.join(str(tmpdir), 'goodartifactshare'))
+ bad_share = create_artifact_share(os.path.join(str(tmpdir), 'badartifactshare'))
+
+ # First build it without the artifact cache configured
+ result = cli.run(project=project, args=['build', 'target.bst'])
+ assert result.exit_code == 0
+
+ # Assert that we are now cached locally
+ state = cli.get_element_state(project, 'target.bst')
+ assert state == 'cached'
+
+ # Configure only the artifact share that we want to avoid.
+ project_conf_file = str(datafiles.join('project.conf'))
+ configure_remote_caches(cli, project_conf_file, [bad_share.repo], [bad_share.repo], [bad_share.repo])
+
+ # Now try bst push
+ result = cli.run(project=project, args=[
+ 'push', 'target.bst', '--remote', good_share.repo
+ ])
+ assert result.exit_code == 0
+ good_share.update_summary()
+ bad_share.update_summary()
+
+ # Assert that all the artifacts are in the share we pushed
+ # to, and not the other.
+ assert_shared(cli, good_share, project, 'target.bst')
+ assert_not_shared(cli, bad_share, project, 'target.bst')
+
+ # Now we've pushed, delete the user's local artifact cache
+ # directory and try to redownload it from the share
+ #
+ artifacts = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifacts)
+
+ result = cli.run(project=project, args=['pull', 'target.bst', '--remote',
+ good_share.repo])
+ assert result.exit_code == 0
+
+ # And assert that it's again in the local cache, without having built
+ assert cli.get_element_state(project, 'target.bst') == 'cached'
+
+
+@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
+@pytest.mark.datafiles(DATA_DIR)
+def test_pull_secondary_cache(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+
+ share1 = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1'))
+ share2 = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2'))
+
+ # First build it without the artifact cache configured
+ result = cli.run(project=project, args=['build', 'target.bst'])
+ assert result.exit_code == 0
+
+ # Assert that we are now cached locally
+ state = cli.get_element_state(project, 'target.bst')
+ assert state == 'cached'
+
+ # bst push to secondary remote
+ result = cli.run(project=project, args=[
+ 'push', 'target.bst', '--remote', share2.repo
+ ])
+ assert result.exit_code == 0
+ share2.update_summary()
+
+ # Now we've pushed, delete the user's local artifact cache
+ artifacts = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifacts)
+
+ # Configure artifact shares
+ project_conf_file = str(datafiles.join('project.conf'))
+ configure_remote_caches(cli, project_conf_file, [], [share1.repo, share2.repo], [])
+
+ # And assert that it's found in share2
+ assert cli.get_element_state(project, 'target.bst') == 'downloadable'
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index 9d897a8..06f53cf 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -3,8 +3,6 @@ import pytest
from tests.testutils import cli, create_artifact_share, configure_remote_caches
from tests.testutils.site import IS_LINUX
-from buildstream import _yaml
-
# Project directory
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
@@ -26,14 +24,16 @@ def assert_shared(cli, share, project, element_name):
@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
@pytest.mark.parametrize(
- 'override_url, project_url, user_url',
+ 'override_urls, project_urls, user_urls',
[
- pytest.param(None, None, 'share.repo', id='user-config'),
- pytest.param(None, 'share.repo', None, id='project-config'),
- pytest.param('share.repo', None, None, id='project-override-in-user-config'),
+ # The leftmost cache is the highest priority one.
+ pytest.param([], [], ['share.repo', '/tmp/do-not-use/user'], id='user-config'),
+ pytest.param([], ['share.repo', '/tmp/do-not-use/project'], ['/tmp/do-not-use/user'], id='project-config'),
+ pytest.param(['share.repo'], ['/tmp/do-not-use/project'], ['/tmp/do-not-use/user'],
+ id='project-override-in-user-config'),
])
@pytest.mark.datafiles(DATA_DIR)
-def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url):
+def test_push(cli, tmpdir, datafiles, user_urls, project_urls, override_urls):
project = str(datafiles)
share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'))
@@ -44,12 +44,13 @@ def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url):
# Assert that we are now cached locally
assert cli.get_element_state(project, 'target.bst') == 'cached'
- override_url = share.repo if override_url == 'share.repo' else override_url
- project_url = share.repo if project_url == 'share.repo' else project_url
- user_url = share.repo if user_url == 'share.repo' else user_url
+ override_urls = [share.repo if url == 'share.repo' else url for url in override_urls]
+ project_urls = [share.repo if url == 'share.repo' else url for url in project_urls]
+ user_urls = [share.repo if url == 'share.repo' else url for url in user_urls]
+ # Configure artifact share
project_conf_file = str(datafiles.join('project.conf'))
- configure_remote_caches(cli, project_conf_file, override_url, project_url, user_url)
+ configure_remote_caches(cli, project_conf_file, override_urls, project_urls, user_urls)
# Now try bst push
result = cli.run(project=project, args=['push', 'target.bst'])
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index ebf38f3..f616e57 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -115,28 +115,47 @@ def create_artifact_share(directory):
# User config is set through a helper on the 'cli' object, while the
# project.conf file is updated manually using the _yaml module.
#
-def configure_remote_caches(cli, project_conf_file, override_url, project_url=None, user_url=None):
+def configure_remote_caches(cli, project_conf_file, override_urls, project_urls=[], user_urls=[]):
user_config = {}
- if user_url is not None:
+ if len(user_urls) == 1:
user_config['artifacts'] = {
- 'url': user_url
+ 'url': user_urls[0]
}
+ elif len(user_urls) > 1:
+ user_config['artifacts'] = [
+ {'url': url} for url in user_urls
+ ]
- if override_url is not None:
+ if len(override_urls) == 1:
user_config['projects'] = {
'test': {
'artifacts': {
- 'url': override_url,
+ 'url': override_urls[0],
}
}
}
+ elif len(override_urls) > 1:
+ user_config['projects'] = {
+ 'test': {
+ 'artifacts': [
+ {'url': override_url} for url in override_urls
+ ]
+ }
+ }
cli.configure(user_config)
- if project_url is not None:
+ if len(project_urls) > 0:
project_config = _yaml.load(project_conf_file)
- project_config.update({
- 'artifacts': {
- 'url': project_url,
- }
- })
+ if len(project_urls) == 1:
+ project_config.update({
+ 'artifacts': {
+ 'url': project_urls[0],
+ }
+ })
+ elif len(project_urls) > 1:
+ project_config.update({
+ 'artifacts': [
+ {'url': url} for url in project_urls
+ ]
+ })
_yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file)
diff --git a/tests/testutils/runcli.py b/tests/testutils/runcli.py
index d1b8c01..27aa21f 100644
--- a/tests/testutils/runcli.py
+++ b/tests/testutils/runcli.py
@@ -290,6 +290,25 @@ class Cli():
result.assert_success()
return result.output.strip()
+ # Strip failure messages that result from pulling from invalid caches.
+ # These are harmless in some cases.
+ #
+ # There are two reasons for this ugly hack. Firstly, click.CliRunner
+ # makes it impossible for us to parse stdout independently of stderr.
+ # This is <https://github.com/pallets/click/issues/371>.
+ #
+ # Secondly, we can't use the normal BuildStream logging at the point
+ # that we need to record failure to contact a configured cache, so
+ # there's no easy way to hide the message. This will change soon and
+ # we should be able to remove this hack once we fix
+ # <https://gitlab.com/BuildStream/buildstream/issues/168>.
+ result_lines = []
+ for line in result.output.split('\n'):
+ if not line.startswith('Failed to fetch remote refs'):
+ result_lines.append(line)
+
+ return '\n'.join(result_lines).strip()
+
# Fetch an element's cache key by invoking bst show
# on the project with the CLI
#
[buildstream] 01/06: tests/frontend: Assert cached state in a
single line
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tristan/multiple-caches
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f4c007e034ff2fbbc9120ae106d53b1ec2dc0f87
Author: Sam Thursfield <sa...@codethink.co.uk>
AuthorDate: Wed Nov 29 11:48:19 2017 +0000
tests/frontend: Assert cached state in a single line
Minor tweak to hopefully make the test cases a bit shorter and more
readable, in preparation for adding more.
---
tests/frontend/pull.py | 18 ++++++------------
tests/frontend/push.py | 6 ++----
2 files changed, 8 insertions(+), 16 deletions(-)
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index 530c1e0..5f06e55 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -43,8 +43,7 @@ def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
result.assert_success()
# Assert that we are now cached locally
- state = cli.get_element_state(project, 'import-bin.bst')
- assert state == 'cached'
+ assert cli.get_element_state(project, 'import-bin.bst') == 'cached'
override_url = share.repo if override_url == 'share.repo' else override_url
project_url = share.repo if project_url == 'share.repo' else project_url
@@ -94,16 +93,14 @@ def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
# Assert that we are now in a downloadable state, nothing
# is cached locally anymore
- state = cli.get_element_state(project, 'import-bin.bst')
- assert state == 'downloadable'
+ assert cli.get_element_state(project, 'import-bin.bst') == 'downloadable'
# Now try bst pull
result = cli.run(project=project, args=['pull', 'import-bin.bst'])
result.assert_success()
# And assert that it's again in the local cache, without having built
- state = cli.get_element_state(project, 'import-bin.bst')
- assert state == 'cached'
+ assert cli.get_element_state(project, 'import-bin.bst') == 'cached'
@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
@@ -117,8 +114,7 @@ def test_push_pull_all(cli, tmpdir, datafiles):
result.assert_success()
# Assert that we are now cached locally
- state = cli.get_element_state(project, 'target.bst')
- assert state == 'cached'
+ assert cli.get_element_state(project, 'target.bst') == 'cached'
# Configure artifact share
cli.configure({
@@ -161,8 +157,7 @@ def test_push_pull_all(cli, tmpdir, datafiles):
# Assert that we are now in a downloadable state, nothing
# is cached locally anymore
for element_name in all_elements:
- state = cli.get_element_state(project, element_name)
- assert state == 'downloadable'
+ assert cli.get_element_state(project, element_name) == 'downloadable'
# Now try bst pull
result = cli.run(project=project, args=['pull', '--deps', 'all', 'target.bst'])
@@ -170,5 +165,4 @@ def test_push_pull_all(cli, tmpdir, datafiles):
# And assert that it's again in the local cache, without having built
for element_name in all_elements:
- state = cli.get_element_state(project, element_name)
- assert state == 'cached'
+ assert cli.get_element_state(project, element_name) == 'cached'
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index 9f8bc5b..b5eddf8 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -42,8 +42,7 @@ def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url):
result.assert_success()
# Assert that we are now cached locally
- state = cli.get_element_state(project, 'target.bst')
- assert state == 'cached'
+ assert cli.get_element_state(project, 'target.bst') == 'cached'
override_url = share.repo if override_url == 'share.repo' else override_url
project_url = share.repo if project_url == 'share.repo' else project_url
@@ -92,8 +91,7 @@ def test_push_all(cli, tmpdir, datafiles):
result.assert_success()
# Assert that we are now cached locally
- state = cli.get_element_state(project, 'target.bst')
- assert state == 'cached'
+ assert cli.get_element_state(project, 'target.bst') == 'cached'
# Configure artifact share
cli.configure({
[buildstream] 02/06: Add support for multiple remote caches
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tristan/multiple-caches
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f3447446606bf3be782f21de10b02d3f11ecc10c
Author: Sam Thursfield <sa...@codethink.co.uk>
AuthorDate: Wed Nov 22 14:58:09 2017 +0000
Add support for multiple remote caches
This extends the 'artifacts' configuration block such that a list of
`url` mappings can be given instead of a single entry. For example:
artifacts:
- url: http://example.com/artifacts1
- url: ssh://ostree@example.com/artifacts2
The OSTreeCache class is updated to set up multiple remotes and query
remote refs from all of them.
There are no automated tests for this yet.
Empty URLs ('') now raise an exception. They cause breakages internally
if we allow them through, and they can only occur if the user or our
tests are misconfiguring things somehow.
We report failure to fetch from the cache by printing a message to
stderr for now. This is because BuildStream's actual logging
functionality can't be used during frontend init -- see issue #168.
---
buildstream/_artifactcache/__init__.py | 2 +-
buildstream/_artifactcache/artifactcache.py | 136 +++++++++++-------
buildstream/_artifactcache/ostreecache.py | 214 +++++++++++++++++-----------
buildstream/_context.py | 9 +-
buildstream/_ostree.py | 2 +-
buildstream/_pipeline.py | 32 ++---
buildstream/_project.py | 5 +-
buildstream/data/userconfig.yaml | 8 --
buildstream/utils.py | 30 ++++
tests/frontend/pull.py | 33 +----
tests/frontend/push.py | 33 +----
tests/testutils/__init__.py | 2 +-
tests/testutils/artifactshare.py | 34 +++++
13 files changed, 317 insertions(+), 223 deletions(-)
diff --git a/buildstream/_artifactcache/__init__.py b/buildstream/_artifactcache/__init__.py
index 991e848..7dc0ce3 100644
--- a/buildstream/_artifactcache/__init__.py
+++ b/buildstream/_artifactcache/__init__.py
@@ -18,4 +18,4 @@
# Authors:
# Tristan Van Berkom <tr...@codethink.co.uk>
-from .artifactcache import ArtifactCache
+from .artifactcache import ArtifactCache, artifact_cache_urls_from_config_node, configured_artifact_cache_urls
diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py
index 1283b37..6eae24b 100644
--- a/buildstream/_artifactcache/artifactcache.py
+++ b/buildstream/_artifactcache/artifactcache.py
@@ -21,12 +21,72 @@
import os
from collections import Mapping
-from .._exceptions import ImplError
+from .._exceptions import ImplError, LoadError, LoadErrorReason
from .. import utils
from .. import _yaml
-# An ArtifactCache manages artifacts
+def artifact_cache_url_from_spec(spec):
+ _yaml.node_validate(spec, ['url'])
+ url = _yaml.node_get(spec, str, 'url')
+ if len(url) == 0:
+ provenance = _yaml.node_get_provenance(spec)
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: empty artifact cache URL".format(provenance))
+ return url
+
+
+# artifact_cache_urls_from_config_node()
+#
+# Parses the configuration of remote artifact caches from a config block.
+#
+# Args:
+# config_node (dict): The config block, which may contain the 'artifacts' key
+#
+# Returns:
+# A list of URLs pointing to remote artifact caches.
+#
+# Raises:
+# LoadError, if the config block contains invalid keys.
+#
+def artifact_cache_urls_from_config_node(config_node):
+ urls = []
+
+ artifacts = config_node.get('artifacts', [])
+ if isinstance(artifacts, Mapping):
+ urls.append(artifact_cache_url_from_spec(artifacts))
+ elif isinstance(artifacts, list):
+ for spec in artifacts:
+ urls.append(artifact_cache_url_from_spec(spec))
+ else:
+ provenance = _yaml.node_get_provenance(config_node, key='artifacts')
+ raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
+ "%s: 'artifacts' must be a single 'url:' mapping, or a list of mappings" %
+ (str(provenance)))
+ return urls
+
+
+# configured_artifact_cache_urls():
+#
+# Return the list of configured artifact remotes for a given project, in priority
+# order. This takes into account the user and project configuration.
+#
+# Args:
+# context (Context): The BuildStream context
+# project (Project): The BuildStream project
+#
+# Returns:
+# A list of URLs pointing to remote artifact caches.
+#
+def configured_artifact_cache_urls(context, project):
+ project_overrides = context._get_overrides(project.name)
+ project_extra_urls = artifact_cache_urls_from_config_node(project_overrides)
+
+ return list(utils.deduplicate(
+ project_extra_urls + project.artifact_urls + context.artifact_urls))
+
+
+# An ArtifactCache manages artifacts.
#
# Args:
# context (Context): The BuildStream context
@@ -36,42 +96,24 @@ class ArtifactCache():
def __init__(self, context, project):
self.context = context
+ self.project = project
os.makedirs(context.artifactdir, exist_ok=True)
self.extractdir = os.path.join(context.artifactdir, 'extract')
self._local = False
+ self.urls = []
- project_overrides = context._get_overrides(project.name)
- artifact_overrides = _yaml.node_get(project_overrides, Mapping, 'artifacts', default_value={})
- override_url = _yaml.node_get(artifact_overrides, str, 'url', default_value='') or None
-
- _yaml.node_validate(artifact_overrides, ['url'])
-
- if override_url:
- self.url = override_url
- elif project.artifact_url:
- self.url = project.artifact_url
- else:
- self.url = context.artifact_url
-
- if self.url:
- if self.url.startswith('/') or self.url.startswith('file://'):
- self._local = True
-
- self.remote = utils.url_directory_name(self.url)
- else:
- self.remote = None
-
- self._offline = False
-
- # initialize_remote():
+ # set_remotes():
#
- # Initialize any remote artifact cache, if needed. This may require network
- # access and could block for several seconds.
+ # Set the list of remote caches, which is initially empty. This will
+ # contact each remote cache.
#
- def initialize_remote(self):
- pass
+ # Args:
+ # urls (list): List of artifact remote URLs, in priority order.
+ # on_failure (callable): Called if we fail to contact one of the caches.
+ def set_remotes(self, urls, on_failure=None):
+ self.urls = urls
# contains():
#
@@ -120,37 +162,28 @@ class ArtifactCache():
raise ImplError("Cache '{kind}' does not implement commit()"
.format(kind=type(self).__name__))
- # set_offline()
+ # has_fetch_remotes():
#
- # Do not attempt to pull or push artifacts.
+ # Check whether any remote repositories are available for fetching.
#
- def set_offline(self):
- self._offline = True
-
- # can_fetch():
+ # Returns: True if any remote repositories are configured, False otherwise
#
- # Check whether remote repository is available for fetching.
- #
- # Returns: True if remote repository is available, False otherwise
- #
- def can_fetch(self):
- return (not self._offline or self._local) and \
- self.remote is not None
+ def has_fetch_remotes(self):
+ return (len(self.urls) > 0)
- # can_push():
+ # has_push_remotes():
#
- # Check whether remote repository is available for pushing.
+ # Check whether any remote repositories are available for pushing.
#
- # Returns: True if remote repository is available, False otherwise
+ # Returns: True if any remote repository is configured, False otherwise
#
- def can_push(self):
- return (not self._offline or self._local) and \
- self.url is not None
+ def has_push_remotes(self):
+ return (len(self.urls) > 0)
# remote_contains_key():
#
# Check whether the artifact for the specified Element is already available
- # in the remote artifact cache.
+ # in any remote artifact cache.
#
# Args:
# element (Element): The Element to check
@@ -160,6 +193,3 @@ class ArtifactCache():
#
def remote_contains(self, element, strength=None):
return False
-
- def fetch_remote_refs(self):
- pass
diff --git a/buildstream/_artifactcache/ostreecache.py b/buildstream/_artifactcache/ostreecache.py
index 78796dc..9987746 100644
--- a/buildstream/_artifactcache/ostreecache.py
+++ b/buildstream/_artifactcache/ostreecache.py
@@ -48,7 +48,7 @@ def buildref(element, key):
return '{0}/{1}/{2}'.format(project.name, element_name, key)
-# An ArtifactCache manages artifacts in an OSTree repository
+# An OSTreeCache manages artifacts in an OSTree repository
#
# Args:
# context (Context): The BuildStream context
@@ -70,35 +70,19 @@ class OSTreeCache(ArtifactCache):
self.repo = _ostree.ensure(ostreedir, False)
self.push_url = None
- self.pull_url = None
-
- self._remote_refs = None
-
- def initialize_remote(self):
- if self.url is not None:
- if self.url.startswith('ssh://'):
- self.push_url = self.url
- try:
- # Contact the remote cache.
- self.pull_url = initialize_push_connection(self.push_url)
- except PushException as e:
- raise ArtifactError("BuildStream did not connect succesfully "
- "to the shared cache: {}".format(e))
- elif self.url.startswith('http://') or self.url.startswith('https://'):
- self.push_url = None
- self.pull_url = self.url
- elif self._local:
- self.push_url = self.url
- self.pull_url = self.url
- else:
- raise ArtifactError("Unsupported URL scheme: {}".format(self.url))
+ self.pull_urls = []
+ self._remote_refs = {}
+
+ def set_remotes(self, urls, on_failure=None):
+ self.urls = urls
+
+ self._initialize_remotes(on_failure)
- _ostree.configure_remote(self.repo, self.remote, self.pull_url)
+ def has_fetch_remotes(self):
+ return (len(self.pull_urls) > 0)
- def can_push(self):
- if self.enable_push:
- return (not self._offline or self._local) and self.push_url is not None
- return False
+ def has_push_remotes(self):
+ return (self.push_url is not None)
# contains():
#
@@ -134,7 +118,7 @@ class OSTreeCache(ArtifactCache):
# Returns: True if the artifact is in the cache, False otherwise
#
def remote_contains_key(self, element, key):
- if not self._remote_refs:
+ if len(self._remote_refs) == 0:
return False
ref = buildref(element, key)
@@ -241,7 +225,7 @@ class OSTreeCache(ArtifactCache):
# pull():
#
- # Pull artifact from remote repository.
+ # Pull artifact from one of the configured remote repositories.
#
# Args:
# element (Element): The Element whose artifact is to be fetched
@@ -249,21 +233,13 @@ class OSTreeCache(ArtifactCache):
#
def pull(self, element, progress=None):
- if self._offline and not self._local:
- raise ArtifactError("Attempt to pull artifact while offline")
-
- if self.pull_url.startswith("/"):
- remote = "file://" + self.pull_url
- else:
- remote = self.remote
-
+ ref = buildref(element, element._get_cache_key())
weak_ref = buildref(element, element._get_cache_key(strength=_KeyStrength.WEAK))
try:
- if self.remote_contains(element, strength=_KeyStrength.STRONG):
+ if ref in self._remote_refs:
# fetch the artifact using the strong cache key
- ref = buildref(element, element._get_cache_key())
- _ostree.fetch(self.repo, remote=remote,
+ _ostree.fetch(self.repo, remote=self._remote_refs[ref],
ref=ref, progress=progress)
# resolve ref to checksum
@@ -271,9 +247,9 @@ class OSTreeCache(ArtifactCache):
# update weak ref by pointing it to this newly fetched artifact
_ostree.set_ref(self.repo, weak_ref, rev)
- elif self.remote_contains(element):
+ elif weak_ref in self._remote_refs:
# fetch the artifact using the weak cache key
- _ostree.fetch(self.repo, remote=remote,
+ _ostree.fetch(self.repo, remote=self._remote_refs[weak_ref],
ref=weak_ref, progress=progress)
# resolve weak_ref to checksum
@@ -292,35 +268,6 @@ class OSTreeCache(ArtifactCache):
raise ArtifactError("Failed to pull artifact for element {}: {}"
.format(element.name, e)) from e
- # fetch_remote_refs():
- #
- # Fetch list of artifacts from remote repository.
- #
- def fetch_remote_refs(self):
- if self.pull_url.startswith("/"):
- remote = "file://" + self.pull_url
- elif self.remote is not None:
- remote = self.remote
- else:
- raise ArtifactError("Attempt to fetch remote refs without any pull URL")
-
- def child_action(repo, remote, q):
- try:
- q.put((True, _ostree.list_remote_refs(self.repo, remote=remote)))
- except OSTreeError as e:
- q.put((False, e))
-
- q = multiprocessing.Queue()
- p = multiprocessing.Process(target=child_action, args=(self.repo, remote, q))
- p.start()
- ret, res = q.get()
- p.join()
-
- if ret:
- self._remote_refs = res
- else:
- raise ArtifactError("Failed to fetch remote refs") from res
-
# push():
#
# Push committed artifact to remote repository.
@@ -336,17 +283,14 @@ class OSTreeCache(ArtifactCache):
# (ArtifactError): if there was an error
def push(self, element):
- if self._offline and not self._local:
- raise ArtifactError("Attempt to push artifact while offline")
-
if self.push_url is None:
raise ArtifactError("The protocol in use does not support pushing.")
ref = buildref(element, element._get_cache_key_from_artifact())
weak_ref = buildref(element, element._get_cache_key(strength=_KeyStrength.WEAK))
- if self.push_url.startswith("/"):
+ if self.push_url.startswith("file://"):
# local repository
- push_repo = _ostree.ensure(self.push_url, True)
+ push_repo = _ostree.ensure(self.push_url[7:], True)
_ostree.fetch(push_repo, remote=self.repo.get_path().get_uri(), ref=ref)
_ostree.fetch(push_repo, remote=self.repo.get_path().get_uri(), ref=weak_ref)
@@ -377,9 +321,117 @@ class OSTreeCache(ArtifactCache):
return pushed
- # set_offline():
+ # _initialize_remote():
+ #
+ # Do protocol-specific initialization necessary to use a given OSTree
+ # remote.
+ #
+ # The SSH protocol that we use only supports pushing so initializing these
+ # involves contacting the remote to find out the corresponding pull URL.
+ #
+ # Args:
+ # url (str): URL of the remote
+ #
+ # Returns:
+ # (str, str): the pull URL and push URL for the remote
+ #
+ # Raises:
+ # ArtifactError: if there was an error
+ def _initialize_remote(self, url):
+ if url.startswith('ssh://'):
+ try:
+ push_url = url
+ pull_url = initialize_push_connection(url)
+ except PushException as e:
+ raise ArtifactError("BuildStream did not connect successfully "
+ "to the shared cache {}: {}".format(url, e))
+ elif url.startswith('/'):
+ push_url = pull_url = 'file://' + url
+ elif url.startswith('file://'):
+ push_url = pull_url = url
+ elif url.startswith('http://') or url.startswith('https://'):
+ push_url = None
+ pull_url = url
+ else:
+ raise ArtifactError("Unsupported URL: {}".format(url))
+
+ return push_url, pull_url
+
+ # _ensure_remote():
#
- # Do not attempt to pull or push artifacts.
+ # Ensure that our OSTree repo has a remote configured for the given URL.
+ # Note that SSH access to remotes is not handled by libostree itself.
#
- def set_offline(self):
- self._offline = True
+ # Args:
+ # repo (OSTree.Repo): an OSTree repository
+ # pull_url (str): the URL where libostree can pull from the remote
+ #
+ # Returns:
+ # (str): the name of the remote, which can be passed to various other
+ # operations implemented by the _ostree module.
+ #
+ # Raises:
+ # OSTreeError: if there was a problem reported by libostree
+ def _ensure_remote(self, repo, pull_url):
+ remote_name = utils.url_directory_name(pull_url)
+ _ostree.configure_remote(repo, remote_name, pull_url)
+ return remote_name
+
+ def _initialize_remotes(self, on_failure=None):
+ self.push_url = None
+ self.pull_urls = []
+ self._remote_refs = {}
+
+ # Callback to initialize one remote in a 'multiprocessing' subprocess.
+ #
+ # We cannot do this in the main process because of the way the tasks
+ # run by the main scheduler calls into libostree using
+ # fork()-without-exec() subprocesses. OSTree fetch operations in
+ # subprocesses hang if fetch operations were previously done in the
+ # main process.
+ #
+ def child_action(url, q):
+ try:
+ push_url, pull_url = self._initialize_remote(url)
+ remote = self._ensure_remote(self.repo, pull_url)
+ remote_refs = _ostree.list_remote_refs(self.repo, remote=remote)
+ q.put((None, push_url, pull_url, remote_refs))
+ except Exception as e:
+ # Exceptions aren't automatically propagated by
+ # multiprocessing, so we catch everything here. Note that
+ # GLib.Error subclasses can't be returned (as they don't
+ # 'pickle') and so they'll be ignored.
+ q.put((e, None, None, None))
+
+ # Kick off all the initialization jobs one by one.
+ #
+ # Note that we cannot use multiprocessing.Pool here because it's not
+ # possible to pickle local functions such as child_action().
+ #
+ q = multiprocessing.Queue()
+ for url in self.urls:
+ p = multiprocessing.Process(target=child_action, args=(url, q))
+ p.start()
+ exception, push_url, pull_url, remote_refs = q.get()
+ p.join()
+
+ if exception and on_failure:
+ on_failure(url, exception)
+ elif exception:
+ raise ArtifactError() from exception
+ else:
+ # Use first pushable remote we find for pushing.
+ if push_url and not self.push_url:
+ self.push_url = push_url
+
+ # Use all pullable remotes, in priority order
+ if pull_url and pull_url not in self.pull_urls:
+ self.pull_urls.append(pull_url)
+
+ # Update our overall map of remote refs with any refs that are
+ # present in the new remote and were not already found in
+ # higher priority ones.
+ remote = self._ensure_remote(self.repo, pull_url)
+ for ref in remote_refs:
+ if ref not in self._remote_refs:
+ self._remote_refs[ref] = remote
diff --git a/buildstream/_context.py b/buildstream/_context.py
index 6d0a19b..78ec9f8 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -29,6 +29,7 @@ from . import _yaml
from ._exceptions import LoadError, LoadErrorReason, BstError
from ._message import Message, MessageType
from ._profile import Topics, profile_start, profile_end
+from ._artifactcache import artifact_cache_urls_from_config_node
# Context()
@@ -61,8 +62,8 @@ class Context():
# The local binary artifact cache directory
self.artifactdir = None
- # The URL from which to push and pull prebuilt artifacts
- self.artifact_url = None
+ # The URLs from which to push and pull prebuilt artifacts
+ self.artifact_urls = []
# The directory to store build logs
self.logdir = None
@@ -161,9 +162,7 @@ class Context():
setattr(self, dir, path)
# Load artifact share configuration
- artifacts = _yaml.node_get(defaults, Mapping, 'artifacts')
- _yaml.node_validate(artifacts, ['url'])
- self.artifact_url = _yaml.node_get(artifacts, str, 'url', default_value='') or None
+ self.artifact_urls = artifact_cache_urls_from_config_node(defaults)
# Load logging config
logging = _yaml.node_get(defaults, Mapping, 'logging')
diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py
index c9f0389..1655563 100644
--- a/buildstream/_ostree.py
+++ b/buildstream/_ostree.py
@@ -357,4 +357,4 @@ def list_remote_refs(repo, remote="origin"):
_, refs = repo.remote_list_refs(remote)
return refs
except GLib.GError as e:
- raise OSTreeError("Failed to fetch remote refs from '{}': {}".format(remote, e.message)) from e
+ raise OSTreeError(message=e.message) from e
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index 5afb42f..5f0c7e4 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -40,6 +40,7 @@ from . import Scope
from . import _site
from . import utils
from ._platform import Platform
+from ._artifactcache import configured_artifact_cache_urls
from ._scheduler import SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
@@ -145,8 +146,8 @@ class Pipeline():
self.initialize_workspaces()
- if use_remote_cache and self.artifacts.can_fetch():
- self.fetch_remote_refs()
+ if use_remote_cache:
+ self.initialize_remote_caches()
self.resolve_cache_keys(inconsistent)
@@ -165,14 +166,13 @@ class Pipeline():
self.project._set_workspace(element, source, workspace)
- def fetch_remote_refs(self):
- with self.timed_activity("Fetching remote refs", silent_nested=True):
- try:
- self.artifacts.initialize_remote()
- self.artifacts.fetch_remote_refs()
- except ArtifactError:
- self.message(MessageType.WARN, "Failed to fetch remote refs")
- self.artifacts.set_offline()
+ def initialize_remote_caches(self):
+ def remote_failed(url, error):
+ self.message(MessageType.WARN, "Failed to fetch remote refs from {}: {}\n".format(url, error))
+
+ with self.timed_activity("Initializing remote caches", silent_nested=True):
+ artifact_urls = configured_artifact_cache_urls(self.context, self.project)
+ self.artifacts.set_remotes(artifact_urls, on_failure=remote_failed)
def resolve_cache_keys(self, inconsistent):
if inconsistent:
@@ -438,12 +438,12 @@ class Pipeline():
if track_plan:
track = TrackQueue(save=save)
queues.append(track)
- if self.artifacts.can_fetch():
+ if self.artifacts.has_fetch_remotes():
pull = PullQueue()
queues.append(pull)
queues.append(fetch)
queues.append(build)
- if self.artifacts.can_push():
+ if self.artifacts.has_push_remotes():
push = PushQueue()
queues.append(push)
@@ -681,8 +681,8 @@ class Pipeline():
#
def pull(self, scheduler, elements):
- if not self.artifacts.can_fetch():
- raise PipelineError("Not configured for pulling artifacts")
+ if not self.artifacts.has_fetch_remotes():
+ raise PipelineError("Not artifact caches available for pulling artifacts")
plan = elements
self.assert_consistent(plan)
@@ -719,8 +719,8 @@ class Pipeline():
#
def push(self, scheduler, elements):
- if not self.artifacts.can_push():
- raise PipelineError("Not configured for pushing artifacts")
+ if not self.artifacts.has_push_remotes():
+ raise PipelineError("No artifact caches available for pushing artifacts")
plan = elements
self.assert_consistent(plan)
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 50f5d9c..35873fb 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -28,6 +28,7 @@ from . import _yaml
from ._profile import Topics, profile_start, profile_end
from ._exceptions import LoadError, LoadErrorReason
from ._options import OptionPool
+from ._artifactcache import artifact_cache_urls_from_config_node
# The base BuildStream format version
@@ -172,9 +173,7 @@ class Project():
#
# Load artifacts pull/push configuration for this project
- artifacts = _yaml.node_get(config, Mapping, 'artifacts', default_value={})
- _yaml.node_validate(artifacts, ['url'])
- self.artifact_url = _yaml.node_get(artifacts, str, 'url', default_value='') or None
+ self.artifact_urls = artifact_cache_urls_from_config_node(config)
# Workspace configurations
self._workspaces = self._load_workspace_config()
diff --git a/buildstream/data/userconfig.yaml b/buildstream/data/userconfig.yaml
index f43989d..edd74ef 100644
--- a/buildstream/data/userconfig.yaml
+++ b/buildstream/data/userconfig.yaml
@@ -48,14 +48,6 @@ scheduler:
#
on-error: quit
-#
-# Artifacts
-#
-artifacts:
-
- # A url from which to push and pull prebuilt artifacts.
- # Some protocols only support pushing.
- url: ''
#
# Logging
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 2b6c451..8cde752 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -34,6 +34,7 @@ import stat
import string
import subprocess
import tempfile
+import itertools
from contextlib import contextmanager
import pkg_resources
@@ -983,3 +984,32 @@ def _glob2re(pat):
else:
res = res + re.escape(c)
return res + '\Z(?ms)'
+
+
+# deduplicate()
+#
+# Remove duplicate entries in a list or other iterable.
+#
+# Copied verbatim from the unique_everseen() example at
+# https://docs.python.org/3/library/itertools.html#itertools-recipes
+#
+# Args:
+# iterable (iterable): What to deduplicate
+# key (callable): Optional function to map from list entry to value
+#
+# Returns:
+# (generator): Generator that produces a deduplicated version of 'iterable'
+#
+def deduplicate(iterable, key=None):
+ seen = set()
+ seen_add = seen.add
+ if key is None:
+ for element in itertools.filterfalse(seen.__contains__, iterable):
+ seen_add(element)
+ yield element
+ else:
+ for element in iterable:
+ k = key(element)
+ if k not in seen:
+ seen_add(k)
+ yield element
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index 5f06e55..18a4b46 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -1,7 +1,7 @@
import os
import shutil
import pytest
-from tests.testutils import cli, create_artifact_share
+from tests.testutils import cli, create_artifact_share, configure_remote_caches
from tests.testutils.site import IS_LINUX
from buildstream import _yaml
@@ -29,9 +29,9 @@ def assert_shared(cli, share, project, element_name):
@pytest.mark.parametrize(
'override_url, project_url, user_url',
[
- pytest.param('', '', 'share.repo', id='user-config'),
- pytest.param('', 'share.repo', '/tmp/share/user', id='project-config'),
- pytest.param('share.repo', '/tmp/share/project', '/tmp/share/user', id='project-override-in-user-config'),
+ pytest.param(None, None, 'share.repo', id='user-config'),
+ pytest.param(None, 'share.repo', None, id='project-config'),
+ pytest.param('share.repo', None, None, id='project-override-in-user-config'),
])
@pytest.mark.datafiles(DATA_DIR)
def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
@@ -49,29 +49,8 @@ def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
project_url = share.repo if project_url == 'share.repo' else project_url
user_url = share.repo if user_url == 'share.repo' else user_url
- # Configure artifact share
- cli.configure({
- 'artifacts': {
- 'url': user_url,
- },
- 'projects': {
- 'test': {
- 'artifacts': {
- 'url': override_url,
- }
- }
- }
- })
-
- if project_url:
- project_conf_file = str(datafiles.join('project.conf'))
- project_config = _yaml.load(project_conf_file)
- project_config.update({
- 'artifacts': {
- 'url': project_url,
- }
- })
- _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file)
+ project_conf_file = str(datafiles.join('project.conf'))
+ configure_remote_caches(cli, project_conf_file, override_url, project_url, user_url)
# Now try bst push
result = cli.run(project=project, args=['push', 'import-bin.bst'])
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index b5eddf8..9d897a8 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -1,6 +1,6 @@
import os
import pytest
-from tests.testutils import cli, create_artifact_share
+from tests.testutils import cli, create_artifact_share, configure_remote_caches
from tests.testutils.site import IS_LINUX
from buildstream import _yaml
@@ -28,9 +28,9 @@ def assert_shared(cli, share, project, element_name):
@pytest.mark.parametrize(
'override_url, project_url, user_url',
[
- pytest.param('', '', 'share.repo', id='user-config'),
- pytest.param('', 'share.repo', '/tmp/share/user', id='project-config'),
- pytest.param('share.repo', '/tmp/share/project', '/tmp/share/user', id='project-override-in-user-config'),
+ pytest.param(None, None, 'share.repo', id='user-config'),
+ pytest.param(None, 'share.repo', None, id='project-config'),
+ pytest.param('share.repo', None, None, id='project-override-in-user-config'),
])
@pytest.mark.datafiles(DATA_DIR)
def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url):
@@ -48,29 +48,8 @@ def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url):
project_url = share.repo if project_url == 'share.repo' else project_url
user_url = share.repo if user_url == 'share.repo' else user_url
- # Configure artifact share
- cli.configure({
- 'artifacts': {
- 'url': user_url,
- },
- 'projects': {
- 'test': {
- 'artifacts': {
- 'url': override_url,
- }
- }
- }
- })
-
- if project_url:
- project_conf_file = str(datafiles.join('project.conf'))
- project_config = _yaml.load(project_conf_file)
- project_config.update({
- 'artifacts': {
- 'url': project_url,
- }
- })
- _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file)
+ project_conf_file = str(datafiles.join('project.conf'))
+ configure_remote_caches(cli, project_conf_file, override_url, project_url, user_url)
# Now try bst push
result = cli.run(project=project, args=['push', 'target.bst'])
diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py
index 9fc450a..f0eb171 100644
--- a/tests/testutils/__init__.py
+++ b/tests/testutils/__init__.py
@@ -1,3 +1,3 @@
from .runcli import cli
from .repo import create_repo, ALL_REPO_KINDS
-from .artifactshare import create_artifact_share
+from .artifactshare import create_artifact_share, configure_remote_caches
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index 907ed76..ebf38f3 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -3,6 +3,8 @@ import pytest
import subprocess
import os
+from buildstream import _yaml
+
from .site import HAVE_OSTREE_CLI
@@ -106,3 +108,35 @@ class ArtifactShare():
def create_artifact_share(directory):
return ArtifactShare(directory)
+
+
+# Write out cache configuration into the user config and project config files.
+#
+# User config is set through a helper on the 'cli' object, while the
+# project.conf file is updated manually using the _yaml module.
+#
+def configure_remote_caches(cli, project_conf_file, override_url, project_url=None, user_url=None):
+ user_config = {}
+ if user_url is not None:
+ user_config['artifacts'] = {
+ 'url': user_url
+ }
+
+ if override_url is not None:
+ user_config['projects'] = {
+ 'test': {
+ 'artifacts': {
+ 'url': override_url,
+ }
+ }
+ }
+ cli.configure(user_config)
+
+ if project_url is not None:
+ project_config = _yaml.load(project_conf_file)
+ project_config.update({
+ 'artifacts': {
+ 'url': project_url,
+ }
+ })
+ _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file)
[buildstream] 03/06: Add `bst push --remote` and `bst pull --remote`
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tristan/multiple-caches
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 536c7e04326d78b213d1f522f5207a63dab59cbc
Author: Sam Thursfield <sa...@codethink.co.uk>
AuthorDate: Tue Dec 5 18:48:16 2017 +0000
Add `bst push --remote` and `bst pull --remote`
This allows pushing and pulling from a specific cache, ignoring what is
configured.
If we choose to add a --remote option to `bst build` in future that
would now be simple to do.
---
buildstream/_frontend/main.py | 30 +++++++++++++++++++++---------
buildstream/_pipeline.py | 25 +++++++++++++++++++------
2 files changed, 40 insertions(+), 15 deletions(-)
diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index 117d148..021d2e2 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -216,7 +216,7 @@ def build(app, elements, all, track, track_save, track_all, track_except):
track = elements
app.initialize(elements, except_=track_except, rewritable=track_save)
- app.pipeline.initialize(use_remote_cache=True, inconsistent=track)
+ app.pipeline.initialize(use_configured_remote_caches=True, inconsistent=track)
app.print_heading()
try:
app.pipeline.build(app.scheduler, all, track, track_save)
@@ -320,12 +320,18 @@ def track(app, elements, deps, except_):
@click.option('--deps', '-d', default='none',
type=click.Choice(['none', 'all']),
help='The dependency artifacts to pull (default: none)')
+@click.option('--remote', '-r',
+ help="The URL of the remote cache (defaults to the first configured cache)")
@click.argument('elements', nargs=-1,
type=click.Path(dir_okay=False, readable=True))
@click.pass_obj
-def pull(app, elements, deps):
+def pull(app, elements, deps, remote):
"""Pull a built artifact from the configured remote artifact cache.
+ By default the artifact will be pulled one of the configured caches
+ if possible, following the usual priority order. If the `--remote` flag
+ is given, only the specified cache will be queried.
+
Specify `--deps` to control which artifacts to pull:
\b
@@ -333,7 +339,8 @@ def pull(app, elements, deps):
all: All dependencies
"""
app.initialize(elements)
- app.pipeline.initialize(use_remote_cache=True)
+ app.pipeline.initialize(use_configured_remote_caches=(remote is None),
+ add_remote_cache=remote)
try:
to_pull = app.pipeline.deps_elements(deps)
app.pipeline.pull(app.scheduler, to_pull)
@@ -351,11 +358,16 @@ def pull(app, elements, deps):
@click.option('--deps', '-d', default='none',
type=click.Choice(['none', 'all']),
help='The dependencies to push (default: none)')
+@click.option('--remote', '-r', default=None,
+ help="The URL of the remote cache (defaults to the first configured cache)")
@click.argument('elements', nargs=-1,
type=click.Path(dir_okay=False, readable=True))
@click.pass_obj
-def push(app, elements, deps):
- """Push a built artifact to the configured remote artifact cache.
+def push(app, elements, deps, remote):
+ """Push a built artifact to a remote artifact cache.
+
+ The default destination is the highest priority configured cache. You can
+ override this by passing a different cache URL with the `--remote` flag.
Specify `--deps` to control which artifacts to push:
@@ -364,7 +376,8 @@ def push(app, elements, deps):
all: All dependencies
"""
app.initialize(elements)
- app.pipeline.initialize(use_remote_cache=True)
+ app.pipeline.initialize(use_configured_remote_caches=(remote is None),
+ add_remote_cache=remote)
try:
to_push = app.pipeline.deps_elements(deps)
app.pipeline.push(app.scheduler, to_push)
@@ -444,7 +457,7 @@ def show(app, elements, deps, except_, order, format, downloadable):
$'---------- %{name} ----------\\n%{vars}'
"""
app.initialize(elements, except_=except_)
- app.pipeline.initialize(use_remote_cache=downloadable)
+ app.pipeline.initialize(use_configured_remote_caches=downloadable)
try:
dependencies = app.pipeline.deps_elements(deps)
except PipelineError as e:
@@ -780,8 +793,7 @@ class App():
#
# Initialize the main pipeline
#
- def initialize(self, elements, except_=tuple(), rewritable=False,
- inconsistent=False, use_remote_cache=False):
+ def initialize(self, elements, except_=tuple(), rewritable=False):
profile_start(Topics.LOAD_PIPELINE, "_".join(t.replace(os.sep, '-') for t in elements))
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index 5f0c7e4..dfcd56a 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -94,7 +94,13 @@ class Planner():
# current source refs will not be the effective refs.
# rewritable (bool): Whether the loaded files should be rewritable
# this is a bit more expensive due to deep copies
-# use_remote_cache (bool): Whether to connect with remote artifact cache
+# use_configured_remote_caches (bool): Whether to connect to configured artifact remotes.
+# add_remote_cache (str): Adds an additional artifact remote URL, which is
+# prepended to the list of remotes (and thus given highest priority).
+#
+# The ticker methods will be called with an element name for each tick, a final
+# tick with None as the argument is passed to signal that processing of this
+# stage has terminated.
#
# Raises:
# LoadError
@@ -137,7 +143,8 @@ class Pipeline():
self.targets = resolved_elements[:len(targets)]
self.exceptions = resolved_elements[len(targets):]
- def initialize(self, use_remote_cache=False, inconsistent=None):
+ def initialize(self, use_configured_remote_caches=False,
+ add_remote_cache=None, inconsistent=None):
# Preflight directly, before ever interrogating caches or
# anything.
self.preflight()
@@ -146,8 +153,15 @@ class Pipeline():
self.initialize_workspaces()
- if use_remote_cache:
- self.initialize_remote_caches()
+ # Initialize remote artifact caches. We allow the commandline to override
+ # the user config in some cases (for example `bst push --remote=...`).
+ artifact_urls = []
+ if add_remote_cache:
+ artifact_urls += [add_remote_cache]
+ if use_configured_remote_caches:
+ artifact_urls += configured_artifact_cache_urls(self.context, self.project)
+ if len(artifact_urls) > 0:
+ self.initialize_remote_caches(artifact_urls)
self.resolve_cache_keys(inconsistent)
@@ -166,12 +180,11 @@ class Pipeline():
self.project._set_workspace(element, source, workspace)
- def initialize_remote_caches(self):
+ def initialize_remote_caches(self, artifact_urls):
def remote_failed(url, error):
self.message(MessageType.WARN, "Failed to fetch remote refs from {}: {}\n".format(url, error))
with self.timed_activity("Initializing remote caches", silent_nested=True):
- artifact_urls = configured_artifact_cache_urls(self.context, self.project)
self.artifacts.set_remotes(artifact_urls, on_failure=remote_failed)
def resolve_cache_keys(self, inconsistent):
[buildstream] 05/06: doc: Document multiple cache support
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tristan/multiple-caches
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 709973f61e15bc4af05a394074ac6fd130604d1d
Author: Sam Thursfield <sa...@codethink.co.uk>
AuthorDate: Thu Jan 4 15:24:24 2018 +0000
doc: Document multiple cache support
---
doc/source/artifacts.rst | 71 +++++++++++++++++++++++++++++++++++-----------
doc/source/config.rst | 27 +++++++++++++++---
doc/source/projectconf.rst | 2 ++
3 files changed, 80 insertions(+), 20 deletions(-)
diff --git a/doc/source/artifacts.rst b/doc/source/artifacts.rst
index 7a5d3c0..8370e06 100644
--- a/doc/source/artifacts.rst
+++ b/doc/source/artifacts.rst
@@ -5,20 +5,59 @@
Artifact Caches
===============
-BuildStream revisions output of each element under it's specific
-cache key in a local artifact cache.
+BuildStream caches the results of builds in a local artifact cache, and will
+avoid building an element if there is a suitable build already present in the
+local artifact cache.
-This artifact cache can however be shared with others, or automated
-builders can be made to contribute to a shared artifact cache so
-that developers dont need to build everything all the time, instead
-they can download prebuilt artifacts from a shared cache, if an artifact
-is available for the specific cache keys they need.
+In addition to the local artifact cache, you can configure one or more remote
+artifact caches and BuildStream will then try to pull a suitable build from one
+of the remotes, falling back to a local build if needed.
-This page outlines how to setup and use a shared artifact cache.
+Configuring BuildStream to use remote caches
+--------------------------------------------
+A project will often set up continuous build infrastructure that pushes
+built artifacts to a shared cache, so developers working on the project can
+make use of these pre-built artifacts instead of having to each build the whole
+project locally. The project can declare this cache in its
+:ref:`project configuration file <project_essentials_artifacts>`.
+Users can declare additional remote caches in the :ref:`user configuration
+<config_artifacts>`. There are several use cases for this: your project may not
+define its own cache, it may be useful to have a local mirror of its cache, or
+you may have a reason to share artifacts privately.
+
+Remote artifact caches are identified by their URL. There are currently three
+supported protocols:
+
+* ``http``: Pull-only access, without transport-layer security
+* ``https``: Pull-only access, with transport-layer security
+* ``ssh``: Push access, authenticated via SSH
+
+BuildStream allows you to configure as many caches as you like, and will query
+them in a specific order:
+
+1. Project-specific overrides in the user config
+2. Project configuration
+3. User configuration
+
+When an artifact is built locally, BuildStream will try to push it to the first
+writable cache in the list. If all the configured caches are pull-only then it
+will not try to push anywhere. You can manually push artifacts to any cache
+using the :ref:`bst pull command <invoking>`.
+
+Artifacts are identified using the element's :ref:`cache key <cachekeys>` so
+the builds provided by a cache should be interchangable with those provided
+by any other cache -- BuildStream doesn't enforce "bit for bit" reproducibility,
+but by controlling all inputs to the build we aim to ensure "functional"
+reproducibility.
+
+
+Setting up a remote artifact cache
+----------------------------------
+The rest of this page outlines how to set up a shared artifact cache.
Setting up the user
--------------------
+~~~~~~~~~~~~~~~~~~~
A specific user is not needed for downloading artifacts, but since we
are going to use ssh to upload the artifacts, you will want a dedicated
user to own the artifact cache.
@@ -29,7 +68,7 @@ user to own the artifact cache.
Installing the receiver
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
You will also need to install BuildStream on the artifact server in order
to receive uploaded artifacts over ssh. Follow the instructions for installing
BuildStream :ref:`here <installing>`
@@ -52,7 +91,7 @@ requiring BuildStream's more exigent dependencies by setting the
Initializing the cache
-----------------------
+~~~~~~~~~~~~~~~~~~~~~~
Now that you have a dedicated user to own the artifact cache, change
to that user, and create the artifact cache ostree repository directly
in it's home directory as such:
@@ -65,7 +104,7 @@ This should result in an artifact cache residing at the path ``/home/artifacts/a
Serve the cache over https
---------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~
This part should be pretty simple, you can do this with various technologies, all
we really require is that you make the artifacts available over https (you can use
http but until we figure out using gpg signed ostree commits for the artifacts, it's
@@ -103,7 +142,7 @@ is the case for hosting anything over https.
Configure and run sshd
-----------------------
+~~~~~~~~~~~~~~~~~~~~~~
You will need to run the sshd service to allow uploading artifacts.
For this you will want something like the following in your ``/etc/ssh/sshd_config``
@@ -129,7 +168,7 @@ For this you will want something like the following in your ``/etc/ssh/sshd_conf
Summary file updates
---------------------
+~~~~~~~~~~~~~~~~~~~~
BuildStream uses the OSTree summary file to determine what artifacts are
available in the remote artifact cache. ``ostree summary -u`` updates
the summary file. This command cannot be run concurrently and thus it
@@ -146,7 +185,7 @@ E.g., create ``/etc/cron.d/artifacts`` with the following content:
User Configuration
-------------------
+~~~~~~~~~~~~~~~~~~
The user configuration for artifacts is documented with the rest
of the :ref:`user configuration documentation <config>`.
@@ -168,7 +207,7 @@ then a user can use the following user configuration:
Authenticating Users
---------------------
+~~~~~~~~~~~~~~~~~~~~
In order to give permission to a given user to upload
artifacts, simply use the regular ``ssh`` method.
diff --git a/doc/source/config.rst b/doc/source/config.rst
index a5ceb87..c69a726 100644
--- a/doc/source/config.rst
+++ b/doc/source/config.rst
@@ -29,12 +29,25 @@ Project Specific Value
The ``projects`` key can be used to specify project specific configurations,
the supported configurations on a project wide basis are listed here.
+.. _config_artifacts:
Artifact Server
~~~~~~~~~~~~~~~
-The artifact server is usually specified by the project you build, but
-it can be overridden on a per project basis using the same format
-:ref:`described here <project_essentials_artifacts>`.
+The project you build will often specify a :ref:`remote artifact cache
+<artifacts>` already, but you may want to specify extra caches. There are two
+ways to do this. You can add one or more global caches:
+
+**Example**
+
+.. code:: yaml
+
+ artifacts:
+ url: https://artifacts.com/artifacts
+
+Caches listed there will be considered lower priority than those specified
+by the project configuration.
+
+You can also add project-specific caches:
**Example**
@@ -43,8 +56,14 @@ it can be overridden on a per project basis using the same format
projects:
project-name:
artifacts:
- url: https://artifacts.com/artifacts
+ - url: https://artifacts.com/artifacts1
+ - url: https://artifacts.com/artifacts2
+
+Caches listed here will be considered higher priority than those specified
+by the project.
+If you give a list of URLs, the last cache in the list will be considered
+to be the highest priority cache.
Strict Build Plan
~~~~~~~~~~~~~~~~~
diff --git a/doc/source/projectconf.rst b/doc/source/projectconf.rst
index c85d381..b3a02e3 100644
--- a/doc/source/projectconf.rst
+++ b/doc/source/projectconf.rst
@@ -78,6 +78,8 @@ with an artifact share.
# A url from which to download prebuilt artifacts
url: https://foo.com/artifacts
+You can also specify a list of caches here; later entries in the list
+will have higher priority than earlier ones.
Plugin Origins and Versions
~~~~~~~~~~~~~~~~~~~~~~~~~~~
[buildstream] 06/06: _artifactcache/ostreecache.py: Handle ^C and
shutdown child task when initializing cache
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tristan/multiple-caches
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0bd2e3b1fa0df81e0f9b00eaee08a877b4499847
Author: Tristan Van Berkom <tr...@codethink.co.uk>
AuthorDate: Thu Jan 11 01:30:59 2018 +0900
_artifactcache/ostreecache.py: Handle ^C and shutdown child task when initializing cache
This fixes issue #141
---
buildstream/_artifactcache/ostreecache.py | 18 ++++++++++++++----
1 file changed, 14 insertions(+), 4 deletions(-)
diff --git a/buildstream/_artifactcache/ostreecache.py b/buildstream/_artifactcache/ostreecache.py
index 9987746..c5178a6 100644
--- a/buildstream/_artifactcache/ostreecache.py
+++ b/buildstream/_artifactcache/ostreecache.py
@@ -20,10 +20,11 @@
import multiprocessing
import os
+import signal
import string
import tempfile
-from .. import _ostree, utils
+from .. import _ostree, _signals, utils
from .._exceptions import ArtifactError
from ..element import _KeyStrength
from .._ostree import OSTreeError
@@ -411,9 +412,18 @@ class OSTreeCache(ArtifactCache):
q = multiprocessing.Queue()
for url in self.urls:
p = multiprocessing.Process(target=child_action, args=(url, q))
- p.start()
- exception, push_url, pull_url, remote_refs = q.get()
- p.join()
+
+ try:
+
+ # Keep SIGINT blocked in the child process
+ with _signals.blocked([signal.SIGINT], ignore=False):
+ p.start()
+
+ exception, push_url, pull_url, remote_refs = q.get()
+ p.join()
+ except KeyboardInterrupt:
+ utils._kill_process_tree(p.pid)
+ raise
if exception and on_failure:
on_failure(url, exception)