You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:23:27 UTC
[buildstream] 16/16: WIP: Fetching obeys numbered suffixes
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4eaa8e3c43ee91a76e6ca40d1064a4f78ad0d354
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Wed May 30 12:52:45 2018 +0100
WIP: Fetching obeys numbered suffixes
---
buildstream/source.py | 93 +++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 83 insertions(+), 10 deletions(-)
diff --git a/buildstream/source.py b/buildstream/source.py
index c0a8820..78a3a1f 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -114,6 +114,18 @@ class SourceError(BstError):
super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason)
+class RefNotFound(SourceError):
+ """This exception should be raised by :class:`.Source` implementations
+ to report to the user that an error occurred because it couldn't find
+ the given ref.
+
+ Args:
+ message (str): The brief error description to report to the user
+ detail (str): A possibly multiline, more detailed error message
+ reason (str): An optional machine readable reason string, used for test cases
+ """
+
+
class Source(Plugin):
"""Source()
@@ -642,16 +654,77 @@ class Source(Plugin):
context = self._get_context()
source_kind = type(self)
for combination in project.generate_alias_combinations(self._used_urls):
- uri_overrides = {k: v[0] for k, v in combination.items()}
- new_source = source_kind(context, project, self.__meta, uri_overrides=uri_overrides)
- new_source._preflight()
- try:
- new_source._fetch()
- except SourceError:
- # SourceErrors from fetch are caused by network error
- # or ref not found
- continue
- return True
+ known_maxima = {}
+ current_suffixes = {}
+ for url, v in combination.items():
+ if v[1]:
+ current_suffixes[url] = 0
+ while True:
+
+ # Compose uri_overrides
+ uri_overrides = {}
+ for url, v in combination.items():
+ if v[1]:
+ uri_overrides[url] = os.path.join(v[0], current_suffixes[url])
+ else:
+ uri_overrides[url] = v[0]
+
+ # Try it
+ new_source = source_kind(context, project, self.__meta,
+ uri_overrides=uri_overrides)
+ new_source._preflight()
+ try:
+ new_source._fetch()
+ except RefNotFound:
+
+ if all([known_maxima.get(url) == current_suffixes[url] for url in current_suffixes]):
+ # Every suffix has been tried without success
+ break
+
+ # repos exist, but a ref wasn't found
+ # Increment current_suffixes
+ increment_suffix = False
+ for url, suffix in current_suffixes.items():
+ if increment_suffix:
+ current_suffixes[url] += 1
+ break
+ if url in known_maxima and suffix == known_maxima[url]:
+ # Maxmimum for this suffix reached, reset to zero and raise the next url
+ # ??? Is it safe to change values of dictionaries as I iterate?
+ current_suffixes[url] = 0
+ increment_suffix = True
+ else:
+ # Don't know it's too high or know it isn't, increment
+ current_suffixes[url] += 1
+ break
+ continue
+ except SourceError:
+
+ # SourceErrors from fetch are caused by network error
+ # or repo not found
+ if not any(current_suffixes.values()):
+ # All suffixes are zero, failure is because this is a bad combination
+ break
+
+ if all([known_maxima.get(url) == current_suffixes[url] for url in current_suffixes]):
+ # Every suffix has been tried without success
+ break
+
+ # One of the suffixes went over. Log the maximum, reset to zero and raise
+ # the next url's suffix.
+ # Current url is the one that doesn't have a maximum defined yet.
+ increment_suffix = False
+ for url in current_suffixes:
+ if increment_suffix:
+ current_suffixes[url] += 1
+ break
+ if url not in known_maxima:
+ known_maxima[url] = current_suffixes[url] - 1
+ current_suffixes[url] = 0
+ increment_suffix = True
+ continue
+
+ return True
#############################################################
# Local Private Methods #