[buildstream] 01/16: project: Parse and store mirrors

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit bb4ace1c60a6b5b327dacc1cfbc8f0401141f4c4
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Mon Apr 9 15:48:50 2018 +0100

    project: Parse and store mirrors
---
 buildstream/_project.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 81 insertions(+), 2 deletions(-)

diff --git a/buildstream/_project.py b/buildstream/_project.py
index 5344e95..d1827d7 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -20,7 +20,7 @@
 
 import os
 import multiprocessing  # for cpu_count()
-from collections import Mapping
+from collections import Mapping, OrderedDict
 from pluginbase import PluginBase
 from . import utils
 from . import _cachekey
@@ -94,6 +94,8 @@ class Project():
         self.base_env_nocache = None             # The base nocache mask (list) for the environment
         self.element_overrides = {}              # Element specific configurations
         self.source_overrides = {}               # Source specific configurations
+        self.mirrors = OrderedDict()             # contains dicts of alias-mappings to URIs.
+        self.default_mirror = None               # The name of the preferred mirror.
 
         #
         # Private Members
@@ -202,6 +204,66 @@ class Project():
         self._assert_plugin_format(source, version)
         return source
 
+    # get_alias_uri()
+    #
+    # Returns the URI for a given alias, if it exists
+    #
+    # Args:
+    #    alias (str): The alias.
+    #
+    # Returns:
+    #    str: The URI for the given alias; or None: if there is no URI for
+    #         that alias.
+    def get_alias_uri(self, alias):
+        return self._aliases.get(alias)
+
+    # generate_alias_combinations()
+    #
+    # Yields every unique combination of mirrors for each alias
+    #
+    # e.g. alias 'foo' has a mirror at 'mirror-A', and the normal alias at 'upstream-A'
+    #      alias 'bar' has no mirror, but does have the normal alias at 'upstream-B'
+    #      We would yield {'foo': 'mirror-A', 'bar': 'upstream-B'},
+    #      and            {'foo': 'upstream-A', 'bar': 'upstream-B'},
+    #
+    # Args:
+    #    aliases (list): A list of aliases to generate combinations for
+    #
+    # Yields:
+    #    a dict mapping aliases to a mirror URI
+    #
+    def generate_alias_combinations(self, aliases):
+        # We numerically address the aliases
+        aliases = list(aliases)
+
+        # Flatten the mirrors and put them in the right order
+        flattened_mirrors = {}
+        for alias in aliases:
+            flattened_mirrors[alias] = []
+            for mirror_location, alias_mappings in self.mirrors.items():
+                if alias in alias_mappings:
+                    mapping_list = list(alias_mappings[alias])
+                    if mirror_location == self.default_mirror:
+                        # The default mirror goes first
+                        flattened_mirrors[alias] = mapping_list + flattened_mirrors[alias]
+                    else:
+                        flattened_mirrors[alias].extend(mapping_list)
+            flattened_mirrors[alias].append(self._aliases[alias])
+
+        combinations = [[]]
+        for alias in aliases:
+            new_combinations = []
+            for x in combinations:
+                for y in flattened_mirrors[alias]:
+                    new_combinations.append(x + [y])
+            combinations = new_combinations
+
+        for combination in combinations:
+            out_combination = {}
+            for i, alias in enumerate(aliases):
+                out_combination[alias] = combination[i]
+            yield out_combination
+
     # _load():
     #
     # Loads the project configuration file in the project directory.
@@ -249,7 +311,7 @@ class Project():
             'aliases', 'name',
             'artifacts', 'options',
             'fail-on-overlap', 'shell',
-            'ref-storage', 'sandbox'
+            'ref-storage', 'sandbox', 'mirrors',
         ])
 
         # The project name, element path and option declarations
@@ -417,6 +479,23 @@ class Project():
 
             self._shell_host_files.append(mount)
 
+        mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
+        default_mirror_set = False
+        for mirror in mirrors:
+            allowed_mirror_fields = [
+                'location-name', 'aliases'
+            ]
+            _yaml.node_validate(mirror, allowed_mirror_fields)
+            mirror_location = _yaml.node_get(mirror, str, 'location-name')
+            alias_mappings = {}
+            for alias_mapping, uris in _yaml.node_items(mirror['aliases']):
+                assert isinstance(uris, list)
+                alias_mappings[alias_mapping] = list(uris)
+            self.mirrors[mirror_location] = alias_mappings
+            if not default_mirror_set:
+                self.default_mirror = mirror_location
+                default_mirror_set = True
+
     # _assert_plugin_format()
     #
     # Helper to raise a PluginError if the loaded plugin is of a lesser version then