[buildstream] 15/16: Improve combination generation, include whether to generate numbered suffixes

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 2827f1160613e4ad6f8af77c12c84a5831decd39
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Fri May 25 15:11:54 2018 +0100

    Improve combination generation, include whether to generate numbered suffixes
---
 buildstream/_mirror.py  |  4 ++--
 buildstream/_project.py | 30 ++++++++++++++++++------------
 buildstream/source.py   |  3 ++-
 3 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/buildstream/_mirror.py b/buildstream/_mirror.py
index d98a5b0..c98529f 100644
--- a/buildstream/_mirror.py
+++ b/buildstream/_mirror.py
@@ -58,7 +58,7 @@ class DefaultMirror(Mirror):
     def get_mirror_uris(self, uri, source):
         url_prefix, url_body = uri.split(utils._ALIAS_SEPARATOR, 1)
         for alias_uri in self.aliases.get(url_prefix, []):
-            yield alias_uri + url_body
+            yield alias_uri + url_body, False
 
 
 class BstGeneratedMirror(Mirror):
@@ -76,4 +76,4 @@ class BstGeneratedMirror(Mirror):
         self.aliases_covered = _yaml.node_get(node, list, 'aliases-covered')
 
     def get_mirror_uris(self, uri, source):
-        yield source.get_normalised_mirror_path(uri, prefix=self.site)
+        yield source.get_normalised_mirror_path(uri, prefix=self.site), True
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 52fd36e..56b644d 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -217,19 +217,22 @@ class Project():
 
     # generate_alias_combinations()
     #
-    # Yields every unique combination of mirrors for each alias
+    # Yields every unique combination of mirrors for each alias, including a flag for
+    # whether a numerical suffix should be appended.
     #
     # e.g. alias 'foo' has a mirror at 'mirror-A', and the normal alias at 'upstream-A'
     #      alias 'bar' has no mirror, but does have the normal alias at 'upstream-B'
-    #      We would yield {'foo': 'mirror-A', 'bar': 'upstream-B'},
-    #      and            {'foo': 'upstream-A', 'bar': 'upstream-B'},
+    #      'mirror-A' should have a numerical suffix appended.
+    #      We would yield {'foo': ('mirror-A', True), 'bar': ('upstream-B', False)},
+    #      and            {'foo': ('upstream-A', False), 'bar': ('upstream-B', False)},
     #
     # Args:
     #    URLs (list): A list of URLs to generate combinations for if they're
     #                 prefixed with an appropriate alias.
     #
     # Yields:
-    #    a dict mapping URLs to a mirrored URL
+    #    a dict mapping URLs to a tuple of the mirrored URL and whether that URL
+    #    should have a numbered suffix appended.
     #
     def generate_alias_combinations(self, urls):
 
@@ -249,26 +252,29 @@ class Project():
         reordered_mirrors = OrderedDict(self.mirrors)
         reordered_mirrors.move_to_end(self.default_mirror, last=False)  # pylint: disable=no-member
 
-        combinations = [[]]
+        combinations = [OrderedDict()]
         for url in urls:
             new_combinations = []
             for combination in combinations:
                 alias = urls_to_aliases[url]
                 for mirror in reordered_mirrors.values():
-                    for uri in mirror.get_mirror_uris(url, self):
-                        new_combinations.append(combination + [uri])
+                    for uri, append_number in mirror.get_mirror_uris(url, self):
+                        new_combination = OrderedDict(combination)
+                        new_combination[url] = (uri, append_number)
+                        new_combinations.append(new_combination)
+
+                # Add the default aliases as valid mirrors
                 if alias in self._aliases:
                     default_alias = self._aliases[alias]
                     _, body = url.split(utils._ALIAS_SEPARATOR, 1)
                     new_url = default_alias + body
-                    new_combinations.append(combination + [new_url])
+                    new_combination = OrderedDict(combination)
+                    new_combination[url] = (new_url, False)
+                    new_combinations.append(new_combination)
             combinations = new_combinations
 
         for combination in combinations:
-            out_combination = {}
-            for i, url in enumerate(urls):
-                out_combination[url] = combination[i]
-            yield out_combination
+            yield combination
 
     # _load():
     #
diff --git a/buildstream/source.py b/buildstream/source.py
index 5f9f02a..c0a8820 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -642,7 +642,8 @@ class Source(Plugin):
         context = self._get_context()
         source_kind = type(self)
         for combination in project.generate_alias_combinations(self._used_urls):
-            new_source = source_kind(context, project, self.__meta, uri_overrides=combination)
+            uri_overrides = {k: v[0] for k, v in combination.items()}
+            new_source = source_kind(context, project, self.__meta, uri_overrides=uri_overrides)
             new_source._preflight()
             try:
                 new_source._fetch()