You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:39:27 UTC
[buildstream] 01/18: project: Parse and store mirrors with default
mirror set via config or command-line
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jonathan/mirror-client-sourcedownloader
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 038ae8ebd0788ce0435799802a203a291b973e8e
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Mon Apr 9 15:48:50 2018 +0100
project: Parse and store mirrors with default mirror set via config or
command-line
In user config (buildstream.conf), it is set with the "default-mirror"
field.
On the command-line, it is set with "--default-mirror"
---
buildstream/_context.py | 6 ++++++
buildstream/_frontend/app.py | 3 ++-
buildstream/_frontend/cli.py | 2 ++
buildstream/_project.py | 41 +++++++++++++++++++++++++++++++++++++---
tests/completions/completions.py | 1 +
5 files changed, 49 insertions(+), 4 deletions(-)
diff --git a/buildstream/_context.py b/buildstream/_context.py
index 1a59af2..a1535c0 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -101,6 +101,9 @@ class Context():
# What to do when a build fails in non interactive mode
self.sched_error_action = 'continue'
+ # The default mirror to fetch from
+ self.default_mirror = None
+
# Whether elements must be rebuilt when their dependencies have changed
self._strict_build_plan = None
@@ -153,6 +156,7 @@ class Context():
_yaml.node_validate(defaults, [
'sourcedir', 'builddir', 'artifactdir', 'logdir',
'scheduler', 'artifacts', 'logging', 'projects',
+ 'default-mirror',
])
for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
@@ -198,6 +202,8 @@ class Context():
# Load per-projects overrides
self._project_overrides = _yaml.node_get(defaults, Mapping, 'projects', default_value={})
+ self.default_mirror = _yaml.node_get(defaults, str, "default-mirror", default_value=None)
+
# Shallow validation of overrides, parts of buildstream which rely
# on the overrides are expected to validate elsewhere.
for _, overrides in _yaml.node_items(self._project_overrides):
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index 4675b0e..77f4b24 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -202,7 +202,8 @@ class App():
# Load the Project
#
try:
- self.project = Project(directory, self.context, cli_options=self._main_options['option'])
+ self.project = Project(directory, self.context, cli_options=self._main_options['option'],
+ default_mirror=self._main_options.get('default_mirror'))
except LoadError as e:
# Let's automatically start a `bst init` session in this case
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 4651245..b52053c 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -217,6 +217,8 @@ def print_version(ctx, param, value):
help="Elements must be rebuilt when their dependencies have changed")
@click.option('--option', '-o', type=click.Tuple([str, str]), multiple=True, metavar='OPTION VALUE',
help="Specify a project option")
+@click.option('--default-mirror', default=None,
+ help="The mirror to fetch from first, before attempting other mirrors")
@click.pass_context
def cli(context, **kwargs):
"""Build and manipulate BuildStream projects
diff --git a/buildstream/_project.py b/buildstream/_project.py
index b568cf8..7cf882b 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -19,7 +19,7 @@
import os
import multiprocessing # for cpu_count()
-from collections import Mapping
+from collections import Mapping, OrderedDict
from pluginbase import PluginBase
from . import utils
from . import _cachekey
@@ -70,7 +70,7 @@ class HostMount():
#
class Project():
- def __init__(self, directory, context, *, junction=None, cli_options=None):
+ def __init__(self, directory, context, *, junction=None, cli_options=None, default_mirror=None):
# The project name
self.name = None
@@ -94,6 +94,9 @@ class Project():
self.base_env_nocache = None # The base nocache mask (list) for the environment
self.element_overrides = {} # Element specific configurations
self.source_overrides = {} # Source specific configurations
+ self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs.
+
+ self.default_mirror = default_mirror or context.default_mirror # The name of the preferred mirror.
#
# Private Members
@@ -202,6 +205,23 @@ class Project():
self._assert_plugin_format(source, version)
return source
+ # get_alias_uris()
+ #
+ # Yields every URI to replace a given alias with
+ def get_alias_uris(self, alias):
+ if not alias or alias not in self._aliases:
+ return [None]
+
+ mirror_list = []
+ for key, alias_mapping in self.mirrors.items():
+ if alias in alias_mapping:
+ if key == self.default_mirror:
+ mirror_list = alias_mapping[alias] + mirror_list
+ else:
+ mirror_list += alias_mapping[alias]
+ mirror_list.append(self._aliases[alias])
+ return mirror_list
+
# _load():
#
# Loads the project configuration file in the project directory.
@@ -249,7 +269,7 @@ class Project():
'aliases', 'name',
'artifacts', 'options',
'fail-on-overlap', 'shell',
- 'ref-storage', 'sandbox'
+ 'ref-storage', 'sandbox', 'mirrors',
])
# The project name, element path and option declarations
@@ -414,6 +434,21 @@ class Project():
self._shell_host_files.append(mount)
+ mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
+ for mirror in mirrors:
+ allowed_mirror_fields = [
+ 'location-name', 'aliases'
+ ]
+ _yaml.node_validate(mirror, allowed_mirror_fields)
+ mirror_location = _yaml.node_get(mirror, str, 'location-name')
+ alias_mappings = {}
+ for alias_mapping, uris in _yaml.node_items(mirror['aliases']):
+ assert isinstance(uris, list)
+ alias_mappings[alias_mapping] = list(uris)
+ self.mirrors[mirror_location] = alias_mappings
+ if not self.default_mirror:
+ self.default_mirror = mirror_location
+
# _assert_plugin_format()
#
# Helper to raise a PluginError if the loaded plugin is of a lesser version then
diff --git a/tests/completions/completions.py b/tests/completions/completions.py
index 7c169c2..1ff026e 100644
--- a/tests/completions/completions.py
+++ b/tests/completions/completions.py
@@ -27,6 +27,7 @@ MAIN_OPTIONS = [
"--colors ",
"--config ",
"--debug ",
+ "--default-mirror ",
"--directory ",
"--error-lines ",
"--fetchers ",