You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:14:30 UTC
[buildstream] 01/16: project: Parse and store mirrors with default
mirror set via config or command-line
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch jonathan/mirror-client-sourcedownloader-tidy
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 105e10200140629203b76a34fbb743e823bf8b74
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Mon Apr 9 15:48:50 2018 +0100
project: Parse and store mirrors with default mirror set via config or
command-line
In user config (buildstream.conf), it is set with the "default-mirror"
field.
On the command-line, it is set with "--default-mirror"
---
buildstream/_context.py | 6 ++++++
buildstream/_frontend/app.py | 3 ++-
buildstream/_frontend/cli.py | 2 ++
buildstream/_project.py | 41 +++++++++++++++++++++++++++++++++++++---
tests/completions/completions.py | 1 +
5 files changed, 49 insertions(+), 4 deletions(-)
diff --git a/buildstream/_context.py b/buildstream/_context.py
index 114ac9e..bd9e458 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -102,6 +102,9 @@ class Context():
# What to do when a build fails in non interactive mode
self.sched_error_action = 'continue'
+ # The default mirror to fetch from
+ self.default_mirror = None
+
# Whether elements must be rebuilt when their dependencies have changed
self._strict_build_plan = None
@@ -154,6 +157,7 @@ class Context():
_yaml.node_validate(defaults, [
'sourcedir', 'builddir', 'artifactdir', 'logdir',
'scheduler', 'artifacts', 'logging', 'projects',
+ 'default-mirror',
])
for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
@@ -199,6 +203,8 @@ class Context():
# Load per-projects overrides
self._project_overrides = _yaml.node_get(defaults, Mapping, 'projects', default_value={})
+ self.default_mirror = _yaml.node_get(defaults, str, "default-mirror", default_value=None)
+
# Shallow validation of overrides, parts of buildstream which rely
# on the overrides are expected to validate elsewhere.
for _, overrides in _yaml.node_items(self._project_overrides):
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index d30b592..8f6dea7 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -203,7 +203,8 @@ class App():
# Load the Project
#
try:
- self.project = Project(directory, self.context, cli_options=self._main_options['option'])
+ self.project = Project(directory, self.context, cli_options=self._main_options['option'],
+ default_mirror=self._main_options.get('default_mirror'))
except LoadError as e:
# Let's automatically start a `bst init` session in this case
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 4651245..b52053c 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -217,6 +217,8 @@ def print_version(ctx, param, value):
help="Elements must be rebuilt when their dependencies have changed")
@click.option('--option', '-o', type=click.Tuple([str, str]), multiple=True, metavar='OPTION VALUE',
help="Specify a project option")
+@click.option('--default-mirror', default=None,
+ help="The mirror to fetch from first, before attempting other mirrors")
@click.pass_context
def cli(context, **kwargs):
"""Build and manipulate BuildStream projects
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 9f42bf6..188bf27 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -20,7 +20,7 @@
import os
import multiprocessing # for cpu_count()
-from collections import Mapping
+from collections import Mapping, OrderedDict
from pluginbase import PluginBase
from . import utils
from . import _cachekey
@@ -71,7 +71,7 @@ class HostMount():
#
class Project():
- def __init__(self, directory, context, *, junction=None, cli_options=None):
+ def __init__(self, directory, context, *, junction=None, cli_options=None, default_mirror=None):
# The project name
self.name = None
@@ -95,6 +95,9 @@ class Project():
self.base_env_nocache = None # The base nocache mask (list) for the environment
self.element_overrides = {} # Element specific configurations
self.source_overrides = {} # Source specific configurations
+ self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs.
+
+ self.default_mirror = default_mirror or context.default_mirror # The name of the preferred mirror.
#
# Private Members
@@ -203,6 +206,23 @@ class Project():
self._assert_plugin_format(source, version)
return source
+ # get_alias_uris()
+ #
+ # Yields every URI to replace a given alias with
+ def get_alias_uris(self, alias):
+ if not alias or alias not in self._aliases:
+ return [None]
+
+ mirror_list = []
+ for key, alias_mapping in self.mirrors.items():
+ if alias in alias_mapping:
+ if key == self.default_mirror:
+ mirror_list = alias_mapping[alias] + mirror_list
+ else:
+ mirror_list += alias_mapping[alias]
+ mirror_list.append(self._aliases[alias])
+ return mirror_list
+
# _load():
#
# Loads the project configuration file in the project directory.
@@ -250,7 +270,7 @@ class Project():
'aliases', 'name',
'artifacts', 'options',
'fail-on-overlap', 'shell',
- 'ref-storage', 'sandbox'
+ 'ref-storage', 'sandbox', 'mirrors',
])
# The project name, element path and option declarations
@@ -415,6 +435,21 @@ class Project():
self._shell_host_files.append(mount)
+ mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
+ for mirror in mirrors:
+ allowed_mirror_fields = [
+ 'location-name', 'aliases'
+ ]
+ _yaml.node_validate(mirror, allowed_mirror_fields)
+ mirror_location = _yaml.node_get(mirror, str, 'location-name')
+ alias_mappings = {}
+ for alias_mapping, uris in _yaml.node_items(mirror['aliases']):
+ assert isinstance(uris, list)
+ alias_mappings[alias_mapping] = list(uris)
+ self.mirrors[mirror_location] = alias_mappings
+ if not self.default_mirror:
+ self.default_mirror = mirror_location
+
# _assert_plugin_format()
#
# Helper to raise a PluginError if the loaded plugin is of a lesser version then
diff --git a/tests/completions/completions.py b/tests/completions/completions.py
index 7c169c2..1ff026e 100644
--- a/tests/completions/completions.py
+++ b/tests/completions/completions.py
@@ -27,6 +27,7 @@ MAIN_OPTIONS = [
"--colors ",
"--config ",
"--debug ",
+ "--default-mirror ",
"--directory ",
"--error-lines ",
"--fetchers ",