You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:23:25 UTC
[buildstream] 14/16: TIDY: Store mirrors as objects, not weird dicts
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b74966f66ca46ad1b5787879b23f53de30e3f198
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Thu May 24 15:37:25 2018 +0100
TIDY: Store mirrors as objects, not weird dicts
---
buildstream/_mirror.py | 79 ++++++++++++++++++++++++++++++++++++++
buildstream/_project.py | 63 +++++++++++++++---------------
buildstream/plugins/sources/git.py | 1 -
buildstream/source.py | 10 +++--
4 files changed, 118 insertions(+), 35 deletions(-)
diff --git a/buildstream/_mirror.py b/buildstream/_mirror.py
new file mode 100644
index 0000000..d98a5b0
--- /dev/null
+++ b/buildstream/_mirror.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2018 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Jonathan Maw <jo...@codethink.co.uk>
+
+
+from . import utils
+from . import _yaml
+from ._exceptions import ImplError, LoadError, LoadErrorReason
+
+
+# MirrorKind()
+#
+# Defines the kind of mirrors that buildstream is capable of handling.
+class MirrorKind():
+
+ # The default type of mirror, replace the alias
+ DEFAULT = 'default'
+
+ # A mirror generated by buildstream
+ BST_GENERATED = 'bst-generated'
+
+
+class Mirror():
+ def __init__(self, node):
+ self.location = _yaml.node_get(node, str, "location-name")
+
+ def get_mirror_uris(self, uri, source):
+ raise ImplError("Base mirror class does not implement get_mirror_uri")
+
+
+class DefaultMirror(Mirror):
+ def __init__(self, node):
+ super().__init__(node)
+ allowed_fields = ['location-name', 'aliases', 'mirror-kind']
+ _yaml.node_validate(node, allowed_fields)
+
+ self.aliases = {}
+ for alias_mapping, uris in _yaml.node_items(node['aliases']):
+ assert isinstance(uris, list)
+ self.aliases[alias_mapping] = list(uris)
+
+ def get_mirror_uris(self, uri, source):
+ url_prefix, url_body = uri.split(utils._ALIAS_SEPARATOR, 1)
+ for alias_uri in self.aliases.get(url_prefix, []):
+ yield alias_uri + url_body
+
+
+class BstGeneratedMirror(Mirror):
+ def __init__(self, node):
+ super().__init__(node)
+ allowed_fields = [
+ 'location-name', 'mirror-kind', 'site', 'aliases-covered'
+ ]
+ _yaml.node_validate(node, allowed_fields)
+ self.site = _yaml.node_get(node, str, 'site')
+ if '://' in self.site:
+ provenance = _yaml.node_get_provenance(node, key='site')
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ '{}: Site should not contain a URI prefix'.format(provenance))
+ self.aliases_covered = _yaml.node_get(node, list, 'aliases-covered')
+
+ def get_mirror_uris(self, uri, source):
+ yield source.get_normalised_mirror_path(uri, prefix=self.site)
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 05e15c7..52fd36e 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -35,6 +35,7 @@ from ._sourcefactory import SourceFactory
from ._projectrefs import ProjectRefs, ProjectRefStorage
from ._versions import BST_FORMAT_VERSION
from ._workspaces import Workspaces
+from ._mirror import MirrorKind, DefaultMirror, BstGeneratedMirror
# HostMount()
@@ -245,30 +246,21 @@ class Project():
# We numerically address urls
url = list(urls)
- # Flatten the mirrors and put them in the right order
- flattened_mirrors = {}
- for alias in aliases:
- flattened_mirrors[alias] = []
- for mirror_location, alias_mappings in self.mirrors.items():
- if alias in alias_mappings:
- mapping_list = list(alias_mappings[alias])
- if mirror_location == self.default_mirror:
- # The default mirror goes first
- flattened_mirrors[alias] = mapping_list + flattened_mirrors[alias]
- else:
- flattened_mirrors[alias].extend(mapping_list)
- flattened_mirrors[alias].append(self._aliases[alias])
+ reordered_mirrors = OrderedDict(self.mirrors)
+ reordered_mirrors.move_to_end(self.default_mirror, last=False) # pylint: disable=no-member
combinations = [[]]
for url in urls:
new_combinations = []
for combination in combinations:
alias = urls_to_aliases[url]
- for mirror_uri in flattened_mirrors[alias]:
- # TODO: MAKE NICE
- _, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
- new_url = mirror_uri + url_body
-
+ for mirror in reordered_mirrors.values():
+ for uri in mirror.get_mirror_uris(url, self):
+ new_combinations.append(combination + [uri])
+ if alias in self._aliases:
+ default_alias = self._aliases[alias]
+ _, body = url.split(utils._ALIAS_SEPARATOR, 1)
+ new_url = default_alias + body
new_combinations.append(combination + [new_url])
combinations = new_combinations
@@ -493,20 +485,29 @@ class Project():
self._shell_host_files.append(mount)
- mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
- for mirror in mirrors:
- allowed_mirror_fields = [
- 'location-name', 'aliases'
- ]
- _yaml.node_validate(mirror, allowed_mirror_fields)
- mirror_location = _yaml.node_get(mirror, str, 'location-name')
- alias_mappings = {}
- for alias_mapping, uris in _yaml.node_items(mirror['aliases']):
- assert isinstance(uris, list)
- alias_mappings[alias_mapping] = list(uris)
- self.mirrors[mirror_location] = alias_mappings
+ self._load_mirrors(config)
+
+ def _load_mirrors(self, config):
+ mirrors_node = _yaml.node_get(config, list, 'mirrors', default_value=[])
+ for mirror_node in mirrors_node:
+ # different kinds of mirror expect different fields
+ mirror_kind = _yaml.node_get(mirror_node, str, 'mirror-kind',
+ default_value=MirrorKind.DEFAULT)
+ if mirror_kind == MirrorKind.DEFAULT:
+ mirror = DefaultMirror(mirror_node)
+ self.mirrors[mirror.location] = mirror
+
+ elif mirror_kind == MirrorKind.BST_GENERATED:
+ mirror = BstGeneratedMirror(mirror_node)
+ self.mirrors[mirror.location] = mirror
+
+ else:
+ provenance = _yaml.node_get_provenance(mirror_node, key='mirror-kind')
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Unexpected mirror-kind: {}".format(provenance, mirror_kind))
+
if not self.default_mirror:
- self.default_mirror = mirror_location
+ self.default_mirror = mirror.location
# _assert_plugin_format()
#
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index 42c5f7e..20393d3 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -107,7 +107,6 @@ class GitMirror():
# system configured tmpdir is not on the same partition.
#
with self.source.tempdir() as tmpdir:
- self.source.info("*** url is '{}'".format(self.url))
self.source.call([self.source.host_git, 'clone', '--mirror', '-n', self.url, tmpdir],
fail="Failed to clone git repository {}".format(self.url))
diff --git a/buildstream/source.py b/buildstream/source.py
index 0428473..5f9f02a 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -304,7 +304,7 @@ class Source(Plugin):
os.makedirs(directory, exist_ok=True)
return directory
- def get_normalised_mirror_path(self, upstream_url, *, prefix="", suffix=""):
+ def get_normalised_mirror_path(self, upstream_url, *, prefix=""):
"""Constructs a path for the mirror from the given URL
Returns:
@@ -313,7 +313,7 @@ class Source(Plugin):
kind = self.get_kind()
normalised_url = utils.url_directory_name(upstream_url)
- return os.path.join(self.__protocol_prefix, prefix, kind, normalised_url, suffix)
+ return os.path.join(self.__protocol_prefix, prefix, kind, normalised_url)
def translate_url(self, url):
"""Translates the given url which may be specified with an alias
@@ -634,8 +634,12 @@ class Source(Plugin):
if not self._used_urls:
return False
- context = self._get_context()
+ # Skip if we have no mirrors defined to fetch from
project = self._get_project()
+ if not project.mirrors:
+ return False
+
+ context = self._get_context()
source_kind = type(self)
for combination in project.generate_alias_combinations(self._used_urls):
new_source = source_kind(context, project, self.__meta, uri_overrides=combination)