You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:23:24 UTC
[buildstream] 13/16: TIDY: Substitute the entire URL,
not just the alias
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7b0015d73eb6409a027990e4ea6f1fe12045264d
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Thu May 17 14:53:48 2018 +0100
TIDY: Substitute the entire URL, not just the alias
---
buildstream/_project.py | 39 +++++++++++++++++++++++++++-----------
buildstream/plugins/sources/git.py | 1 +
buildstream/source.py | 35 ++++++++++++++++++++++------------
3 files changed, 52 insertions(+), 23 deletions(-)
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 90a73d5..05e15c7 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -224,14 +224,26 @@ class Project():
# and {'foo': 'upstream-A', 'bar': 'upstream-B'},
#
# Args:
- # aliases (list): A list of aliases to generate combinations for
+ # URLs (list): A list of URLs to generate combinations for if they're
+ # prefixed with an appropriate alias.
#
# Yields:
- # a dict mapping aliases to a mirror URI
+ # a dict mapping URLs to a mirrored URL
#
- def generate_alias_combinations(self, aliases):
- # We numerically address the aliases
- aliases = list(aliases)
+ def generate_alias_combinations(self, urls):
+
+ aliases = set()
+ urls_to_aliases = {}
+ # Generate the aliases
+ for url in urls:
+ if utils._ALIAS_SEPARATOR in url:
+ url_alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
+ if url_alias in self._aliases:
+ aliases.add(url_alias)
+ urls_to_aliases[url] = url_alias
+
+ # We numerically address urls
+ url = list(urls)
# Flatten the mirrors and put them in the right order
flattened_mirrors = {}
@@ -248,17 +260,22 @@ class Project():
flattened_mirrors[alias].append(self._aliases[alias])
combinations = [[]]
- for alias in aliases:
+ for url in urls:
new_combinations = []
- for x in combinations:
- for y in flattened_mirrors[alias]:
- new_combinations.append(x + [y])
+ for combination in combinations:
+ alias = urls_to_aliases[url]
+ for mirror_uri in flattened_mirrors[alias]:
+ # TODO: MAKE NICE
+ _, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
+ new_url = mirror_uri + url_body
+
+ new_combinations.append(combination + [new_url])
combinations = new_combinations
for combination in combinations:
out_combination = {}
- for i, alias in enumerate(aliases):
- out_combination[alias] = combination[i]
+ for i, url in enumerate(urls):
+ out_combination[url] = combination[i]
yield out_combination
# _load():
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index 20393d3..42c5f7e 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -107,6 +107,7 @@ class GitMirror():
# system configured tmpdir is not on the same partition.
#
with self.source.tempdir() as tmpdir:
+ self.source.info("*** url is '{}'".format(self.url))
self.source.call([self.source.host_git, 'clone', '--mirror', '-n', self.url, tmpdir],
fail="Failed to clone git repository {}".format(self.url))
diff --git a/buildstream/source.py b/buildstream/source.py
index 9c70b00..0428473 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -124,8 +124,9 @@ class Source(Plugin):
"""
__defaults = {} # The defaults from the project
__defaults_set = False # Flag, in case there are not defaults at all
+ __protocol_prefix = "http://" # Default URI prefix that most source protocols accept
- def __init__(self, context, project, meta, *, alias_overrides=None):
+ def __init__(self, context, project, meta, *, uri_overrides=None):
provenance = _yaml.node_get_provenance(meta.config)
super().__init__("{}-{}".format(meta.element_name, meta.element_index),
context, project, provenance, "source")
@@ -135,8 +136,8 @@ class Source(Plugin):
self.__element_kind = meta.element_kind # The kind of the element owning this source
self.__directory = meta.directory # Staging relative directory
self.__consistency = Consistency.INCONSISTENT # Cached consistency state
- self.__alias_overrides = alias_overrides # Aliases to use instead of the one from the project
- self._expected_aliases = set() # A hacky way to store which aliases the source used
+ self.__uri_overrides = uri_overrides # URIs to use directly instead of resolving aliases
+ self._used_urls = set() # A hacky way of tracking which URLs were used.
self.__meta = meta # MetaSource stored so we can copy this source later.
# Collect the composited element configuration and
@@ -303,6 +304,17 @@ class Source(Plugin):
os.makedirs(directory, exist_ok=True)
return directory
+ def get_normalised_mirror_path(self, upstream_url, *, prefix="", suffix=""):
+ """Constructs a path for the mirror from the given URL
+
+ Returns:
+ (str): The path for the mirror
+ """
+
+ kind = self.get_kind()
+ normalised_url = utils.url_directory_name(upstream_url)
+ return os.path.join(self.__protocol_prefix, prefix, kind, normalised_url, suffix)
+
def translate_url(self, url):
"""Translates the given url which may be specified with an alias
into a fully qualified url.
@@ -313,20 +325,19 @@ class Source(Plugin):
Returns:
str: The fully qualified url, with aliases resolved
"""
- if self.__alias_overrides:
- if url and utils._ALIAS_SEPARATOR in url:
- url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
- url = self.__alias_overrides[url_alias] + url_body
+ if self.__uri_overrides:
+ if url:
+ url = self.__uri_overrides[url]
return url
else:
project = self._get_project()
- # Sneakily store the alias
+ # Sneakily store the URL if it uses an alias.
if url and utils._ALIAS_SEPARATOR in url:
url_alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
# The alias must already be defined in the project's aliases
# otherwise http://foo gets treated like it contains an alias
if project.get_alias_uri(url_alias):
- self._expected_aliases.add(url_alias)
+ self._used_urls.add(url)
return project.translate_url(url)
@@ -620,14 +631,14 @@ class Source(Plugin):
#
def _mirrored_fetch(self):
# Mirrors can't do anything if this source doesn't use aliases
- if not self._expected_aliases:
+ if not self._used_urls:
return False
context = self._get_context()
project = self._get_project()
source_kind = type(self)
- for combination in project.generate_alias_combinations(self._expected_aliases):
- new_source = source_kind(context, project, self.__meta, alias_overrides=combination)
+ for combination in project.generate_alias_combinations(self._used_urls):
+ new_source = source_kind(context, project, self.__meta, uri_overrides=combination)
new_source._preflight()
try:
new_source._fetch()