You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 08:21:59 UTC
[buildstream] branch jonathan/mirror-client created (now 4eaa8e3)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 4eaa8e3 WIP: Fetching obeys numbered suffixes
This branch includes the following new commits:
new bb4ace1 project: Parse and store mirrors
new 7040439 Move _ALIAS_SEPARATOR into utils
new 2b72504 source: Store the url aliases or use an override
new 463199d bzr.py: Improve mirror support
new c5db5de testutils: Add a helper to copy a testutils repo
new d699c81 tests: Add a very rough test for fetching
new 277cae9 git.py: Improve mirror support
new 3ce7919 When fetching, try to fetch from mirrors first
new d41bbc5 tests: Add a test for fetching when a source uses multiple aliases
new 77424cd tests: Add default-mirror tests
new 2cbe8e8 Fix parse and store
new 32b110e Set default mirror via command-line or user config
new 7b0015d TIDY: Substitute the entire URL, not just the alias
new b74966f TIDY: Store mirrors as objects, not weird dicts
new 2827f11 Improve combination generation, include whether to generate numbered suffixes
new 4eaa8e3 WIP: Fetching obeys numbered suffixes
The 16 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 10/16: tests: Add default-mirror tests
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 77424cdb28130ea3405cf6cedcfd1b411b7eac50
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Wed May 16 14:12:30 2018 +0100
tests: Add default-mirror tests
---
tests/frontend/mirror.py | 206 +++++++++++++++++++++++++++++++++++++----------
1 file changed, 164 insertions(+), 42 deletions(-)
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
index 9fdbd38..592e499 100644
--- a/tests/frontend/mirror.py
+++ b/tests/frontend/mirror.py
@@ -11,6 +11,74 @@ TOP_DIR = os.path.dirname(os.path.realpath(__file__))
DATA_DIR = os.path.join(TOP_DIR, 'project')
+def generate_element(output_file):
+ element = {
+ 'kind': 'import',
+ 'sources': [
+ {
+ 'kind': 'fetch_source',
+ "output-text": output_file,
+ "urls": ["foo:repo1", "bar:repo2"],
+ "fetch-succeeds": {
+ "FOO/repo1": True,
+ "BAR/repo2": False,
+ "OOF/repo1": False,
+ "RAB/repo2": True,
+ "OFO/repo1": False,
+ "RBA/repo2": False,
+ "ooF/repo1": False,
+ "raB/repo2": False,
+ }
+ }
+ ]
+ }
+ return element
+
+
+def generate_project():
+ project = {
+ 'name': 'test',
+ 'element-path': 'elements',
+ 'aliases': {
+ 'foo': 'FOO/',
+ 'bar': 'BAR/',
+ },
+ 'mirrors': [
+ {
+ 'location-name': 'middle-earth',
+ 'aliases': {
+ 'foo': ['OOF/'],
+ 'bar': ['RAB/'],
+ },
+ },
+ {
+ 'location-name': 'arrakis',
+ 'aliases': {
+ 'foo': ['OFO/'],
+ 'bar': ['RBA/'],
+ },
+ },
+ {
+ 'location-name': 'oz',
+ 'aliases': {
+ 'foo': ['ooF/'],
+ 'bar': ['raB/'],
+ }
+ },
+ ],
+ 'plugins': [
+ {
+ 'origin': 'local',
+ 'path': 'sources',
+ 'sources': {
+ 'fetch_source': 0
+ }
+ }
+ ]
+ }
+ return project
+
+
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
def test_mirror_fetch(cli, tmpdir, datafiles, kind):
@@ -78,51 +146,11 @@ def test_mirror_fetch_multi(cli, tmpdir, datafiles):
os.makedirs(element_dir, exist_ok=True)
element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
- element = {
- 'kind': 'import',
- 'sources': [
- {
- 'kind': 'fetch_source',
- "output-text": output_file,
- "urls": ["foo:repo1", "bar:repo2"],
- "fetch-succeeds": {
- "FOO/repo1": True,
- "BAR/repo2": False,
- "OOF/repo1": False,
- "RAB/repo2": True
- }
- }
- ]
- }
+ element = generate_element(output_file)
_yaml.dump(element, element_path)
project_file = os.path.join(project_dir, 'project.conf')
- project = {
- 'name': 'test',
- 'element-path': 'elements',
- 'aliases': {
- "foo": "FOO/",
- "bar": "BAR/"
- },
- 'mirrors': [
- {
- 'location-name': 'middle-earth',
- 'aliases': {
- "foo": ["OOF/"],
- "bar": ["RAB/"]
- },
- },
- ],
- 'plugins': [
- {
- 'origin': 'local',
- 'path': 'sources',
- 'sources': {
- 'fetch_source': 0
- }
- }
- ]
- }
+ project = generate_project()
_yaml.dump(project, project_file)
result = cli.run(project=project_dir, args=['fetch', element_name])
@@ -131,3 +159,97 @@ def test_mirror_fetch_multi(cli, tmpdir, datafiles):
contents = f.read()
assert "Fetch foo:repo1 succeeded from FOO/repo1" in contents
assert "Fetch bar:repo2 succeeded from RAB/repo2" in contents
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_fetch_default_cmdline(cli, tmpdir, datafiles):
+ output_file = os.path.join(str(tmpdir), "output.txt")
+ project_dir = str(tmpdir)
+ element_dir = os.path.join(project_dir, 'elements')
+ os.makedirs(element_dir, exist_ok=True)
+ element_name = "test.bst"
+ element_path = os.path.join(element_dir, element_name)
+ element = generate_element(output_file)
+ _yaml.dump(element, element_path)
+
+ project_file = os.path.join(project_dir, 'project.conf')
+ project = generate_project()
+ _yaml.dump(project, project_file)
+
+ result = cli.run(project=project_dir, args=['--default-mirror', 'arrakis', 'fetch', element_name])
+ result.assert_success()
+ with open(output_file) as f:
+ contents = f.read()
+ print(contents)
+ # Success if fetching from arrakis' mirror happened before middle-earth's
+ arrakis_str = "OFO/repo1"
+ arrakis_pos = contents.find(arrakis_str)
+ assert arrakis_pos != -1, "'{}' wasn't found".format(arrakis_str)
+ me_str = "OOF/repo1"
+ me_pos = contents.find(me_str)
+ assert me_pos != -1, "'{}' wasn't found".format(me_str)
+ assert arrakis_pos < me_pos, "'{}' wasn't found before '{}'".format(arrakis_str, me_str)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_fetch_default_userconfig(cli, tmpdir, datafiles):
+ output_file = os.path.join(str(tmpdir), "output.txt")
+ project_dir = str(tmpdir)
+ element_dir = os.path.join(project_dir, 'elements')
+ os.makedirs(element_dir, exist_ok=True)
+ element_name = "test.bst"
+ element_path = os.path.join(element_dir, element_name)
+ element = generate_element(output_file)
+ _yaml.dump(element, element_path)
+
+ project_file = os.path.join(project_dir, 'project.conf')
+ project = generate_project()
+ _yaml.dump(project, project_file)
+
+ cli.configure({'default-mirror': 'oz'})
+
+ result = cli.run(project=project_dir, args=['fetch', element_name])
+ result.assert_success()
+ with open(output_file) as f:
+ contents = f.read()
+ print(contents)
+ # Success if fetching from Oz' mirror happened before middle-earth's
+ oz_str = "ooF/repo1"
+ oz_pos = contents.find(oz_str)
+ assert oz_pos != -1, "'{}' wasn't found".format(oz_str)
+ me_str = "OOF/repo1"
+ me_pos = contents.find(me_str)
+ assert me_pos != -1, "'{}' wasn't found".format(me_str)
+ assert oz_pos < me_pos, "'{}' wasn't found before '{}'".format(oz_str, me_str)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_fetch_default_cmdline_overrides_config(cli, tmpdir, datafiles):
+ output_file = os.path.join(str(tmpdir), "output.txt")
+ project_dir = str(tmpdir)
+ element_dir = os.path.join(project_dir, 'elements')
+ os.makedirs(element_dir, exist_ok=True)
+ element_name = "test.bst"
+ element_path = os.path.join(element_dir, element_name)
+ element = generate_element(output_file)
+ _yaml.dump(element, element_path)
+
+ project_file = os.path.join(project_dir, 'project.conf')
+ project = generate_project()
+ _yaml.dump(project, project_file)
+
+ cli.configure({'default-mirror': 'oz'})
+
+ result = cli.run(project=project_dir, args=['--default-mirror', 'arrakis', 'fetch', element_name])
+ result.assert_success()
+ with open(output_file) as f:
+ contents = f.read()
+ print(contents)
+ # Success if fetching from arrakis' mirror happened before middle-earth's
+ arrakis_str = "OFO/repo1"
+ arrakis_pos = contents.find(arrakis_str)
+ assert arrakis_pos != -1, "'{}' wasn't found".format(arrakis_str)
+ me_str = "OOF/repo1"
+ me_pos = contents.find(me_str)
+ assert me_pos != -1, "'{}' wasn't found".format(me_str)
+ assert arrakis_pos < me_pos, "'{}' wasn't found before '{}'".format(arrakis_str, me_str)
[buildstream] 16/16: WIP: Fetching obeys numbered suffixes
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4eaa8e3c43ee91a76e6ca40d1064a4f78ad0d354
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Wed May 30 12:52:45 2018 +0100
WIP: Fetching obeys numbered suffixes
---
buildstream/source.py | 93 +++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 83 insertions(+), 10 deletions(-)
diff --git a/buildstream/source.py b/buildstream/source.py
index c0a8820..78a3a1f 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -114,6 +114,18 @@ class SourceError(BstError):
super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason)
+class RefNotFound(SourceError):
+ """This exception should be raised by :class:`.Source` implementations
+ to report to the user that an error occurred because it couldn't find
+ the given ref.
+
+ Args:
+ message (str): The brief error description to report to the user
+ detail (str): A possibly multiline, more detailed error message
+ reason (str): An optional machine readable reason string, used for test cases
+ """
+
+
class Source(Plugin):
"""Source()
@@ -642,16 +654,77 @@ class Source(Plugin):
context = self._get_context()
source_kind = type(self)
for combination in project.generate_alias_combinations(self._used_urls):
- uri_overrides = {k: v[0] for k, v in combination.items()}
- new_source = source_kind(context, project, self.__meta, uri_overrides=uri_overrides)
- new_source._preflight()
- try:
- new_source._fetch()
- except SourceError:
- # SourceErrors from fetch are caused by network error
- # or ref not found
- continue
- return True
+ known_maxima = {}
+ current_suffixes = {}
+ for url, v in combination.items():
+ if v[1]:
+ current_suffixes[url] = 0
+ while True:
+
+ # Compose uri_overrides
+ uri_overrides = {}
+ for url, v in combination.items():
+ if v[1]:
+ uri_overrides[url] = os.path.join(v[0], current_suffixes[url])
+ else:
+ uri_overrides[url] = v[0]
+
+ # Try it
+ new_source = source_kind(context, project, self.__meta,
+ uri_overrides=uri_overrides)
+ new_source._preflight()
+ try:
+ new_source._fetch()
+ except RefNotFound:
+
+ if all([known_maxima.get(url) == current_suffixes[url] for url in current_suffixes]):
+ # Every suffix has been tried without success
+ break
+
+ # repos exist, but a ref wasn't found
+ # Increment current_suffixes
+ increment_suffix = False
+ for url, suffix in current_suffixes.items():
+ if increment_suffix:
+ current_suffixes[url] += 1
+ break
+ if url in known_maxima and suffix == known_maxima[url]:
+ # Maxmimum for this suffix reached, reset to zero and raise the next url
+ # ??? Is it safe to change values of dictionaries as I iterate?
+ current_suffixes[url] = 0
+ increment_suffix = True
+ else:
+ # Don't know it's too high or know it isn't, increment
+ current_suffixes[url] += 1
+ break
+ continue
+ except SourceError:
+
+ # SourceErrors from fetch are caused by network error
+ # or repo not found
+ if not any(current_suffixes.values()):
+ # All suffixes are zero, failure is because this is a bad combination
+ break
+
+ if all([known_maxima.get(url) == current_suffixes[url] for url in current_suffixes]):
+ # Every suffix has been tried without success
+ break
+
+ # One of the suffixes went over. Log the maximum, reset to zero and raise
+ # the next url's suffix.
+ # Current url is the one that doesn't have a maximum defined yet.
+ increment_suffix = False
+ for url in current_suffixes:
+ if increment_suffix:
+ current_suffixes[url] += 1
+ break
+ if url not in known_maxima:
+ known_maxima[url] = current_suffixes[url] - 1
+ current_suffixes[url] = 0
+ increment_suffix = True
+ continue
+
+ return True
#############################################################
# Local Private Methods #
[buildstream] 03/16: source: Store the url aliases or use an
override
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2b725046ae71534004f20736c4492acf2794b05d
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Mon Apr 9 16:20:05 2018 +0100
source: Store the url aliases or use an override
This is part of a later plan to implement mirroring without forcing
everyone to update their sources. We use the expected calls to
Source.translate_url() when running Source.configure() to extract the
aliases from the URL. Multiple aliases must be extracted because
sources exist that may fetch from multiple aliases (for example, git
submodules)
Later, we want to substitute another URI where the alias normally reads
from the project - We accomplish this by re-instantiating the Source
with the alias overrides passed as an argument to the constructor.
---
buildstream/source.py | 22 +++++++++++++++++++---
1 file changed, 19 insertions(+), 3 deletions(-)
diff --git a/buildstream/source.py b/buildstream/source.py
index fa547d6..48803ee 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -125,7 +125,7 @@ class Source(Plugin):
__defaults = {} # The defaults from the project
__defaults_set = False # Flag, in case there are not defaults at all
- def __init__(self, context, project, meta):
+ def __init__(self, context, project, meta, *, alias_overrides=None):
provenance = _yaml.node_get_provenance(meta.config)
super().__init__("{}-{}".format(meta.element_name, meta.element_index),
context, project, provenance, "source")
@@ -135,6 +135,8 @@ class Source(Plugin):
self.__element_kind = meta.element_kind # The kind of the element owning this source
self.__directory = meta.directory # Staging relative directory
self.__consistency = Consistency.INCONSISTENT # Cached consistency state
+ self.__alias_overrides = alias_overrides # Aliases to use instead of the one from the project
+ self._expected_aliases = set() # A hacky way to store which aliases the source used
# Collect the composited element configuration and
# ask the element to configure itself.
@@ -310,8 +312,22 @@ class Source(Plugin):
Returns:
str: The fully qualified url, with aliases resolved
"""
- project = self._get_project()
- return project.translate_url(url)
+ if self.__alias_overrides:
+ if url and utils._ALIAS_SEPARATOR in url:
+ url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
+ url = self.__alias_overrides[url_alias] + url_body
+ return url
+ else:
+ project = self._get_project()
+ # Sneakily store the alias
+ if url and utils._ALIAS_SEPARATOR in url:
+ url_alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
+ # The alias must already be defined in the project's aliases
+ # otherwise http://foo gets treated like it contains an alias
+ if project.get_alias_uri(url_alias):
+ self._expected_aliases.add(url_alias)
+
+ return project.translate_url(url)
def get_project_directory(self):
"""Fetch the project base directory
[buildstream] 15/16: Improve combination generation,
include whether to generate numbered suffixes
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2827f1160613e4ad6f8af77c12c84a5831decd39
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Fri May 25 15:11:54 2018 +0100
Improve combination generation, include whether to generate numbered suffixes
---
buildstream/_mirror.py | 4 ++--
buildstream/_project.py | 30 ++++++++++++++++++------------
buildstream/source.py | 3 ++-
3 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/buildstream/_mirror.py b/buildstream/_mirror.py
index d98a5b0..c98529f 100644
--- a/buildstream/_mirror.py
+++ b/buildstream/_mirror.py
@@ -58,7 +58,7 @@ class DefaultMirror(Mirror):
def get_mirror_uris(self, uri, source):
url_prefix, url_body = uri.split(utils._ALIAS_SEPARATOR, 1)
for alias_uri in self.aliases.get(url_prefix, []):
- yield alias_uri + url_body
+ yield alias_uri + url_body, False
class BstGeneratedMirror(Mirror):
@@ -76,4 +76,4 @@ class BstGeneratedMirror(Mirror):
self.aliases_covered = _yaml.node_get(node, list, 'aliases-covered')
def get_mirror_uris(self, uri, source):
- yield source.get_normalised_mirror_path(uri, prefix=self.site)
+ yield source.get_normalised_mirror_path(uri, prefix=self.site), True
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 52fd36e..56b644d 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -217,19 +217,22 @@ class Project():
# generate_alias_combinations()
#
- # Yields every unique combination of mirrors for each alias
+ # Yields every unique combination of mirrors for each alias, including a flag for
+ # whether a numerical suffix should be appended.
#
# e.g. alias 'foo' has a mirror at 'mirror-A', and the normal alias at 'upstream-A'
# alias 'bar' has no mirror, but does have the normal alias at 'upstream-B'
- # We would yield {'foo': 'mirror-A', 'bar': 'upstream-B'},
- # and {'foo': 'upstream-A', 'bar': 'upstream-B'},
+ # 'mirror-A' should have a numerical suffix appended.
+ # We would yield {'foo': ('mirror-A', True), 'bar': ('upstream-B', False)},
+ # and {'foo': ('upstream-A', False), 'bar': ('upstream-B', False)},
#
# Args:
# URLs (list): A list of URLs to generate combinations for if they're
# prefixed with an appropriate alias.
#
# Yields:
- # a dict mapping URLs to a mirrored URL
+ # a dict mapping URLs to a tuple of the mirrored URL and whether that URL
+ # should have a numbered suffix appended.
#
def generate_alias_combinations(self, urls):
@@ -249,26 +252,29 @@ class Project():
reordered_mirrors = OrderedDict(self.mirrors)
reordered_mirrors.move_to_end(self.default_mirror, last=False) # pylint: disable=no-member
- combinations = [[]]
+ combinations = [OrderedDict()]
for url in urls:
new_combinations = []
for combination in combinations:
alias = urls_to_aliases[url]
for mirror in reordered_mirrors.values():
- for uri in mirror.get_mirror_uris(url, self):
- new_combinations.append(combination + [uri])
+ for uri, append_number in mirror.get_mirror_uris(url, self):
+ new_combination = OrderedDict(combination)
+ new_combination[url] = (uri, append_number)
+ new_combinations.append(new_combination)
+
+ # Add the default aliases as valid mirrors
if alias in self._aliases:
default_alias = self._aliases[alias]
_, body = url.split(utils._ALIAS_SEPARATOR, 1)
new_url = default_alias + body
- new_combinations.append(combination + [new_url])
+ new_combination = OrderedDict(combination)
+ new_combination[url] = (new_url, False)
+ new_combinations.append(new_combination)
combinations = new_combinations
for combination in combinations:
- out_combination = {}
- for i, url in enumerate(urls):
- out_combination[url] = combination[i]
- yield out_combination
+ yield combination
# _load():
#
diff --git a/buildstream/source.py b/buildstream/source.py
index 5f9f02a..c0a8820 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -642,7 +642,8 @@ class Source(Plugin):
context = self._get_context()
source_kind = type(self)
for combination in project.generate_alias_combinations(self._used_urls):
- new_source = source_kind(context, project, self.__meta, uri_overrides=combination)
+ uri_overrides = {k: v[0] for k, v in combination.items()}
+ new_source = source_kind(context, project, self.__meta, uri_overrides=uri_overrides)
new_source._preflight()
try:
new_source._fetch()
[buildstream] 08/16: When fetching, try to fetch from mirrors first
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3ce7919f68f0ec427764298dbc279e1ceb421f21
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Thu Apr 12 16:37:28 2018 +0100
When fetching, try to fetch from mirrors first
**KLUDGE WARNING**: This involves making the source store its "meta"
object so that it's possible to create a copy of the source inside the
fetch queue, instead of back when the pipeline was being loaded.
---
buildstream/_scheduler/fetchqueue.py | 6 +++++-
buildstream/source.py | 27 +++++++++++++++++++++++++++
2 files changed, 32 insertions(+), 1 deletion(-)
diff --git a/buildstream/_scheduler/fetchqueue.py b/buildstream/_scheduler/fetchqueue.py
index 6105572..33e36f3 100644
--- a/buildstream/_scheduler/fetchqueue.py
+++ b/buildstream/_scheduler/fetchqueue.py
@@ -41,7 +41,11 @@ class FetchQueue(Queue):
def process(self, element):
for source in element.sources():
- source._fetch()
+ # Try to fetch from the mirrors first
+ source_fetched = source._mirrored_fetch()
+ # Fall back to the default
+ if not source_fetched:
+ source._fetch()
def status(self, element):
# state of dependencies may have changed, recalculate element state
diff --git a/buildstream/source.py b/buildstream/source.py
index 48803ee..9c70b00 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -137,6 +137,7 @@ class Source(Plugin):
self.__consistency = Consistency.INCONSISTENT # Cached consistency state
self.__alias_overrides = alias_overrides # Aliases to use instead of the one from the project
self._expected_aliases = set() # A hacky way to store which aliases the source used
+ self.__meta = meta # MetaSource stored so we can copy this source later.
# Collect the composited element configuration and
# ask the element to configure itself.
@@ -610,6 +611,32 @@ class Source(Plugin):
return new_ref
+ # _mirrored_fetch():
+ #
+ # Tries to fetch from every mirror, stopping once it succeeds
+ #
+ # Returns:
+ # (bool): True if it successfully fetched from a mirror.
+ #
+ def _mirrored_fetch(self):
+ # Mirrors can't do anything if this source doesn't use aliases
+ if not self._expected_aliases:
+ return False
+
+ context = self._get_context()
+ project = self._get_project()
+ source_kind = type(self)
+ for combination in project.generate_alias_combinations(self._expected_aliases):
+ new_source = source_kind(context, project, self.__meta, alias_overrides=combination)
+ new_source._preflight()
+ try:
+ new_source._fetch()
+ except SourceError:
+ # SourceErrors from fetch are caused by network error
+ # or ref not found
+ continue
+ return True
+
#############################################################
# Local Private Methods #
#############################################################
[buildstream] 09/16: tests: Add a test for fetching when a source
uses multiple aliases
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit d41bbc5c68c225c4c821d8ce8be3dff38476f049
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Thu May 3 17:30:50 2018 +0100
tests: Add a test for fetching when a source uses multiple aliases
---
tests/frontend/mirror.py | 63 ++++++++++++++++++++++
tests/frontend/project/sources/fetch_source.py | 73 ++++++++++++++++++++++++++
2 files changed, 136 insertions(+)
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
index b45aa29..9fdbd38 100644
--- a/tests/frontend/mirror.py
+++ b/tests/frontend/mirror.py
@@ -68,3 +68,66 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
# But at least we can be sure it succeeds
result = cli.run(project=project_dir, args=['fetch', element_name])
result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_fetch_multi(cli, tmpdir, datafiles):
+ output_file = os.path.join(str(tmpdir), "output.txt")
+ project_dir = str(tmpdir)
+ element_dir = os.path.join(project_dir, 'elements')
+ os.makedirs(element_dir, exist_ok=True)
+ element_name = "test.bst"
+ element_path = os.path.join(element_dir, element_name)
+ element = {
+ 'kind': 'import',
+ 'sources': [
+ {
+ 'kind': 'fetch_source',
+ "output-text": output_file,
+ "urls": ["foo:repo1", "bar:repo2"],
+ "fetch-succeeds": {
+ "FOO/repo1": True,
+ "BAR/repo2": False,
+ "OOF/repo1": False,
+ "RAB/repo2": True
+ }
+ }
+ ]
+ }
+ _yaml.dump(element, element_path)
+
+ project_file = os.path.join(project_dir, 'project.conf')
+ project = {
+ 'name': 'test',
+ 'element-path': 'elements',
+ 'aliases': {
+ "foo": "FOO/",
+ "bar": "BAR/"
+ },
+ 'mirrors': [
+ {
+ 'location-name': 'middle-earth',
+ 'aliases': {
+ "foo": ["OOF/"],
+ "bar": ["RAB/"]
+ },
+ },
+ ],
+ 'plugins': [
+ {
+ 'origin': 'local',
+ 'path': 'sources',
+ 'sources': {
+ 'fetch_source': 0
+ }
+ }
+ ]
+ }
+ _yaml.dump(project, project_file)
+
+ result = cli.run(project=project_dir, args=['fetch', element_name])
+ result.assert_success()
+ with open(output_file) as f:
+ contents = f.read()
+ assert "Fetch foo:repo1 succeeded from FOO/repo1" in contents
+ assert "Fetch bar:repo2 succeeded from RAB/repo2" in contents
diff --git a/tests/frontend/project/sources/fetch_source.py b/tests/frontend/project/sources/fetch_source.py
new file mode 100644
index 0000000..81b9777
--- /dev/null
+++ b/tests/frontend/project/sources/fetch_source.py
@@ -0,0 +1,73 @@
+import os
+import sys
+
+from buildstream import Source, Consistency, SourceError
+
+# Expected config
+# sources:
+# - output-text: $FILE
+# urls:
+# - foo:bar
+# - baz:quux
+# fetch-succeeds:
+# Foo/bar: true
+# ooF/bar: false
+
+
+class FetchSource(Source):
+ # Read config to know which URLs to fetch
+ def configure(self, node):
+ self.original_urls = self.node_get_member(node, list, 'urls')
+ self.urls = [self.translate_url(url) for url in self.original_urls]
+ self.output_file = self.node_get_member(node, str, 'output-text')
+ self.fetch_succeeds = {}
+ if 'fetch-succeeds' in node:
+ self.fetch_succeeds = {x[0]: x[1] for x in self.node_items(node['fetch-succeeds'])}
+ self.urls_cached = False
+
+ def preflight(self):
+ output_dir = os.path.dirname(self.output_file)
+ if not os.path.exists(output_dir):
+ raise SourceError("Directory '{}' does not exist".format(output_dir))
+
+ def get_unique_key(self):
+ return {"urls": self.original_urls, "output_file": self.output_file}
+
+ def get_consistency(self):
+ if not os.path.exists(self.output_file):
+ return Consistency.RESOLVED
+
+ all_fetched = True
+ with open(self.output_file, "r") as f:
+ contents = f.read()
+ for url in self.original_urls:
+ if url not in contents:
+ return Consistency.RESOLVED
+
+ return Consistency.CACHED
+
+ # We dont have a ref, we're a local file...
+ def load_ref(self, node):
+ pass
+
+ def get_ref(self):
+ return None # pragma: nocover
+
+ def set_ref(self, ref, node):
+ pass # pragma: nocover
+
+ def fetch(self):
+ with open(self.output_file, "a") as f:
+ for i, url in enumerate(self.urls):
+ origin_url = self.original_urls[i]
+ success = url in self.fetch_succeeds and self.fetch_succeeds[url]
+ message = "Fetch {} {} from {}\n".format(origin_url,
+ "succeeded" if success else "failed",
+ url)
+ f.write(message)
+ if not success:
+ raise SourceError("Failed to fetch {}".format(url))
+
+
+def setup():
+ return FetchSource
[buildstream] 06/16: tests: Add a very rough test for fetching
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit d699c8153aeb8f4de1b117a7a61923cfca52107e
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Wed Apr 11 17:06:07 2018 +0100
tests: Add a very rough test for fetching
---
tests/frontend/mirror.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 70 insertions(+)
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
new file mode 100644
index 0000000..b45aa29
--- /dev/null
+++ b/tests/frontend/mirror.py
@@ -0,0 +1,70 @@
+import os
+import pytest
+
+from tests.testutils import cli, create_repo, ALL_REPO_KINDS
+
+from buildstream import _yaml
+
+
+# Project directory
+TOP_DIR = os.path.dirname(os.path.realpath(__file__))
+DATA_DIR = os.path.join(TOP_DIR, 'project')
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_mirror_fetch(cli, tmpdir, datafiles, kind):
+ bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+ dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+ upstream_repodir = os.path.join(str(tmpdir), 'upstream')
+ mirror_repodir = os.path.join(str(tmpdir), 'mirror')
+ project_dir = os.path.join(str(tmpdir), 'project')
+ os.makedirs(project_dir)
+ element_dir = os.path.join(project_dir, 'elements')
+
+ # Create repo objects of the upstream and mirror
+ upstream_repo = create_repo(kind, upstream_repodir)
+ upstream_ref = upstream_repo.create(bin_files_path)
+ mirror_repo = upstream_repo.copy(mirror_repodir)
+ mirror_ref = upstream_ref
+ upstream_ref = upstream_repo.create(dev_files_path)
+
+ element = {
+ 'kind': 'import',
+ 'sources': [
+ upstream_repo.source_config(ref=upstream_ref)
+ ]
+ }
+ element_name = 'test.bst'
+ element_path = os.path.join(element_dir, element_name)
+ full_repo = element['sources'][0]['url']
+ upstream_map, repo_name = os.path.split(full_repo)
+ alias = 'foo-' + kind
+ aliased_repo = alias + ':' + repo_name
+ element['sources'][0]['url'] = aliased_repo
+ mirror_map, _ = os.path.split(mirror_repo.repo)
+ os.makedirs(element_dir)
+ _yaml.dump(element, element_path)
+
+ project = {
+ 'name': 'test',
+ 'element-path': 'elements',
+ 'aliases': {
+ alias: upstream_map + "/"
+ },
+ 'mirrors': [
+ {
+ 'location-name': 'middle-earth',
+ 'aliases': {
+ alias: ["file://" + mirror_map + "/"],
+ },
+ },
+ ]
+ }
+ project_file = os.path.join(project_dir, 'project.conf')
+ _yaml.dump(project, project_file)
+
+ # No obvious ways of checking that the mirror has been fetched
+ # But at least we can be sure it succeeds
+ result = cli.run(project=project_dir, args=['fetch', element_name])
+ result.assert_success()
[buildstream] 14/16: TIDY: Store mirrors as objects, not weird dicts
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b74966f66ca46ad1b5787879b23f53de30e3f198
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Thu May 24 15:37:25 2018 +0100
TIDY: Store mirrors as objects, not weird dicts
---
buildstream/_mirror.py | 79 ++++++++++++++++++++++++++++++++++++++
buildstream/_project.py | 63 +++++++++++++++---------------
buildstream/plugins/sources/git.py | 1 -
buildstream/source.py | 10 +++--
4 files changed, 118 insertions(+), 35 deletions(-)
diff --git a/buildstream/_mirror.py b/buildstream/_mirror.py
new file mode 100644
index 0000000..d98a5b0
--- /dev/null
+++ b/buildstream/_mirror.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2018 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Jonathan Maw <jo...@codethink.co.uk>
+
+
+from . import utils
+from . import _yaml
+from ._exceptions import ImplError, LoadError, LoadErrorReason
+
+
+# MirrorKind()
+#
+# Defines the kind of mirrors that buildstream is capable of handling.
+class MirrorKind():
+
+ # The default type of mirror, replace the alias
+ DEFAULT = 'default'
+
+ # A mirror generated by buildstream
+ BST_GENERATED = 'bst-generated'
+
+
+class Mirror():
+ def __init__(self, node):
+ self.location = _yaml.node_get(node, str, "location-name")
+
+ def get_mirror_uris(self, uri, source):
+ raise ImplError("Base mirror class does not implement get_mirror_uri")
+
+
+class DefaultMirror(Mirror):
+ def __init__(self, node):
+ super().__init__(node)
+ allowed_fields = ['location-name', 'aliases', 'mirror-kind']
+ _yaml.node_validate(node, allowed_fields)
+
+ self.aliases = {}
+ for alias_mapping, uris in _yaml.node_items(node['aliases']):
+ assert isinstance(uris, list)
+ self.aliases[alias_mapping] = list(uris)
+
+ def get_mirror_uris(self, uri, source):
+ url_prefix, url_body = uri.split(utils._ALIAS_SEPARATOR, 1)
+ for alias_uri in self.aliases.get(url_prefix, []):
+ yield alias_uri + url_body
+
+
+class BstGeneratedMirror(Mirror):
+ def __init__(self, node):
+ super().__init__(node)
+ allowed_fields = [
+ 'location-name', 'mirror-kind', 'site', 'aliases-covered'
+ ]
+ _yaml.node_validate(node, allowed_fields)
+ self.site = _yaml.node_get(node, str, 'site')
+ if '://' in self.site:
+ provenance = _yaml.node_get_provenance(node, key='site')
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ '{}: Site should not contain a URI prefix'.format(provenance))
+ self.aliases_covered = _yaml.node_get(node, list, 'aliases-covered')
+
+ def get_mirror_uris(self, uri, source):
+ yield source.get_normalised_mirror_path(uri, prefix=self.site)
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 05e15c7..52fd36e 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -35,6 +35,7 @@ from ._sourcefactory import SourceFactory
from ._projectrefs import ProjectRefs, ProjectRefStorage
from ._versions import BST_FORMAT_VERSION
from ._workspaces import Workspaces
+from ._mirror import MirrorKind, DefaultMirror, BstGeneratedMirror
# HostMount()
@@ -245,30 +246,21 @@ class Project():
# We numerically address urls
url = list(urls)
- # Flatten the mirrors and put them in the right order
- flattened_mirrors = {}
- for alias in aliases:
- flattened_mirrors[alias] = []
- for mirror_location, alias_mappings in self.mirrors.items():
- if alias in alias_mappings:
- mapping_list = list(alias_mappings[alias])
- if mirror_location == self.default_mirror:
- # The default mirror goes first
- flattened_mirrors[alias] = mapping_list + flattened_mirrors[alias]
- else:
- flattened_mirrors[alias].extend(mapping_list)
- flattened_mirrors[alias].append(self._aliases[alias])
+ reordered_mirrors = OrderedDict(self.mirrors)
+ reordered_mirrors.move_to_end(self.default_mirror, last=False) # pylint: disable=no-member
combinations = [[]]
for url in urls:
new_combinations = []
for combination in combinations:
alias = urls_to_aliases[url]
- for mirror_uri in flattened_mirrors[alias]:
- # TODO: MAKE NICE
- _, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
- new_url = mirror_uri + url_body
-
+ for mirror in reordered_mirrors.values():
+ for uri in mirror.get_mirror_uris(url, self):
+ new_combinations.append(combination + [uri])
+ if alias in self._aliases:
+ default_alias = self._aliases[alias]
+ _, body = url.split(utils._ALIAS_SEPARATOR, 1)
+ new_url = default_alias + body
new_combinations.append(combination + [new_url])
combinations = new_combinations
@@ -493,20 +485,29 @@ class Project():
self._shell_host_files.append(mount)
- mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
- for mirror in mirrors:
- allowed_mirror_fields = [
- 'location-name', 'aliases'
- ]
- _yaml.node_validate(mirror, allowed_mirror_fields)
- mirror_location = _yaml.node_get(mirror, str, 'location-name')
- alias_mappings = {}
- for alias_mapping, uris in _yaml.node_items(mirror['aliases']):
- assert isinstance(uris, list)
- alias_mappings[alias_mapping] = list(uris)
- self.mirrors[mirror_location] = alias_mappings
+ self._load_mirrors(config)
+
+ def _load_mirrors(self, config):
+ mirrors_node = _yaml.node_get(config, list, 'mirrors', default_value=[])
+ for mirror_node in mirrors_node:
+ # different kinds of mirror expect different fields
+ mirror_kind = _yaml.node_get(mirror_node, str, 'mirror-kind',
+ default_value=MirrorKind.DEFAULT)
+ if mirror_kind == MirrorKind.DEFAULT:
+ mirror = DefaultMirror(mirror_node)
+ self.mirrors[mirror.location] = mirror
+
+ elif mirror_kind == MirrorKind.BST_GENERATED:
+ mirror = BstGeneratedMirror(mirror_node)
+ self.mirrors[mirror.location] = mirror
+
+ else:
+ provenance = _yaml.node_get_provenance(mirror_node, key='mirror-kind')
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Unexpected mirror-kind: {}".format(provenance, mirror_kind))
+
if not self.default_mirror:
- self.default_mirror = mirror_location
+ self.default_mirror = mirror.location
# _assert_plugin_format()
#
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index 42c5f7e..20393d3 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -107,7 +107,6 @@ class GitMirror():
# system configured tmpdir is not on the same partition.
#
with self.source.tempdir() as tmpdir:
- self.source.info("*** url is '{}'".format(self.url))
self.source.call([self.source.host_git, 'clone', '--mirror', '-n', self.url, tmpdir],
fail="Failed to clone git repository {}".format(self.url))
diff --git a/buildstream/source.py b/buildstream/source.py
index 0428473..5f9f02a 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -304,7 +304,7 @@ class Source(Plugin):
os.makedirs(directory, exist_ok=True)
return directory
- def get_normalised_mirror_path(self, upstream_url, *, prefix="", suffix=""):
+ def get_normalised_mirror_path(self, upstream_url, *, prefix=""):
"""Constructs a path for the mirror from the given URL
Returns:
@@ -313,7 +313,7 @@ class Source(Plugin):
kind = self.get_kind()
normalised_url = utils.url_directory_name(upstream_url)
- return os.path.join(self.__protocol_prefix, prefix, kind, normalised_url, suffix)
+ return os.path.join(self.__protocol_prefix, prefix, kind, normalised_url)
def translate_url(self, url):
"""Translates the given url which may be specified with an alias
@@ -634,8 +634,12 @@ class Source(Plugin):
if not self._used_urls:
return False
- context = self._get_context()
+ # Skip if we have no mirrors defined to fetch from
project = self._get_project()
+ if not project.mirrors:
+ return False
+
+ context = self._get_context()
source_kind = type(self)
for combination in project.generate_alias_combinations(self._used_urls):
new_source = source_kind(context, project, self.__meta, uri_overrides=combination)
[buildstream] 13/16: TIDY: Substitute the entire URL,
not just the alias
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7b0015d73eb6409a027990e4ea6f1fe12045264d
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Thu May 17 14:53:48 2018 +0100
TIDY: Substitute the entire URL, not just the alias
---
buildstream/_project.py | 39 +++++++++++++++++++++++++++-----------
buildstream/plugins/sources/git.py | 1 +
buildstream/source.py | 35 ++++++++++++++++++++++------------
3 files changed, 52 insertions(+), 23 deletions(-)
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 90a73d5..05e15c7 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -224,14 +224,26 @@ class Project():
# and {'foo': 'upstream-A', 'bar': 'upstream-B'},
#
# Args:
- # aliases (list): A list of aliases to generate combinations for
+ # URLs (list): A list of URLs to generate combinations for if they're
+ # prefixed with an appropriate alias.
#
# Yields:
- # a dict mapping aliases to a mirror URI
+ # a dict mapping URLs to a mirrored URL
#
- def generate_alias_combinations(self, aliases):
- # We numerically address the aliases
- aliases = list(aliases)
+ def generate_alias_combinations(self, urls):
+
+ aliases = set()
+ urls_to_aliases = {}
+ # Generate the aliases
+ for url in urls:
+ if utils._ALIAS_SEPARATOR in url:
+ url_alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
+ if url_alias in self._aliases:
+ aliases.add(url_alias)
+ urls_to_aliases[url] = url_alias
+
+ # We numerically address urls
+ url = list(urls)
# Flatten the mirrors and put them in the right order
flattened_mirrors = {}
@@ -248,17 +260,22 @@ class Project():
flattened_mirrors[alias].append(self._aliases[alias])
combinations = [[]]
- for alias in aliases:
+ for url in urls:
new_combinations = []
- for x in combinations:
- for y in flattened_mirrors[alias]:
- new_combinations.append(x + [y])
+ for combination in combinations:
+ alias = urls_to_aliases[url]
+ for mirror_uri in flattened_mirrors[alias]:
+ # TODO: MAKE NICE
+ _, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
+ new_url = mirror_uri + url_body
+
+ new_combinations.append(combination + [new_url])
combinations = new_combinations
for combination in combinations:
out_combination = {}
- for i, alias in enumerate(aliases):
- out_combination[alias] = combination[i]
+ for i, url in enumerate(urls):
+ out_combination[url] = combination[i]
yield out_combination
# _load():
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index 20393d3..42c5f7e 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -107,6 +107,7 @@ class GitMirror():
# system configured tmpdir is not on the same partition.
#
with self.source.tempdir() as tmpdir:
+ self.source.info("*** url is '{}'".format(self.url))
self.source.call([self.source.host_git, 'clone', '--mirror', '-n', self.url, tmpdir],
fail="Failed to clone git repository {}".format(self.url))
diff --git a/buildstream/source.py b/buildstream/source.py
index 9c70b00..0428473 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -124,8 +124,9 @@ class Source(Plugin):
"""
__defaults = {} # The defaults from the project
__defaults_set = False # Flag, in case there are not defaults at all
+ __protocol_prefix = "http://" # Default URI prefix that most source protocols accept
- def __init__(self, context, project, meta, *, alias_overrides=None):
+ def __init__(self, context, project, meta, *, uri_overrides=None):
provenance = _yaml.node_get_provenance(meta.config)
super().__init__("{}-{}".format(meta.element_name, meta.element_index),
context, project, provenance, "source")
@@ -135,8 +136,8 @@ class Source(Plugin):
self.__element_kind = meta.element_kind # The kind of the element owning this source
self.__directory = meta.directory # Staging relative directory
self.__consistency = Consistency.INCONSISTENT # Cached consistency state
- self.__alias_overrides = alias_overrides # Aliases to use instead of the one from the project
- self._expected_aliases = set() # A hacky way to store which aliases the source used
+ self.__uri_overrides = uri_overrides # URIs to use directly instead of resolving aliases
+ self._used_urls = set() # A hacky way of tracking which URLs were used.
self.__meta = meta # MetaSource stored so we can copy this source later.
# Collect the composited element configuration and
@@ -303,6 +304,17 @@ class Source(Plugin):
os.makedirs(directory, exist_ok=True)
return directory
+ def get_normalised_mirror_path(self, upstream_url, *, prefix="", suffix=""):
+ """Constructs a path for the mirror from the given URL
+
+ Returns:
+ (str): The path for the mirror
+ """
+
+ kind = self.get_kind()
+ normalised_url = utils.url_directory_name(upstream_url)
+ return os.path.join(self.__protocol_prefix, prefix, kind, normalised_url, suffix)
+
def translate_url(self, url):
"""Translates the given url which may be specified with an alias
into a fully qualified url.
@@ -313,20 +325,19 @@ class Source(Plugin):
Returns:
str: The fully qualified url, with aliases resolved
"""
- if self.__alias_overrides:
- if url and utils._ALIAS_SEPARATOR in url:
- url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
- url = self.__alias_overrides[url_alias] + url_body
+ if self.__uri_overrides:
+ if url:
+ url = self.__uri_overrides[url]
return url
else:
project = self._get_project()
- # Sneakily store the alias
+ # Sneakily store the URL if it uses an alias.
if url and utils._ALIAS_SEPARATOR in url:
url_alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
# The alias must already be defined in the project's aliases
# otherwise http://foo gets treated like it contains an alias
if project.get_alias_uri(url_alias):
- self._expected_aliases.add(url_alias)
+ self._used_urls.add(url)
return project.translate_url(url)
@@ -620,14 +631,14 @@ class Source(Plugin):
#
def _mirrored_fetch(self):
# Mirrors can't do anything if this source doesn't use aliases
- if not self._expected_aliases:
+ if not self._used_urls:
return False
context = self._get_context()
project = self._get_project()
source_kind = type(self)
- for combination in project.generate_alias_combinations(self._expected_aliases):
- new_source = source_kind(context, project, self.__meta, alias_overrides=combination)
+ for combination in project.generate_alias_combinations(self._used_urls):
+ new_source = source_kind(context, project, self.__meta, uri_overrides=combination)
new_source._preflight()
try:
new_source._fetch()
[buildstream] 02/16: Move _ALIAS_SEPARATOR into utils
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 70404399ea36f6969c1742182f9611269df12b4f
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Mon Apr 9 16:18:27 2018 +0100
Move _ALIAS_SEPARATOR into utils
The separator is useful in source files other than _project.py
---
buildstream/_project.py | 8 ++------
buildstream/utils.py | 4 ++++
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/buildstream/_project.py b/buildstream/_project.py
index d1827d7..b23a5a8 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -37,10 +37,6 @@ from ._versions import BST_FORMAT_VERSION
from ._workspaces import Workspaces
-# The separator we use for user specified aliases
-_ALIAS_SEPARATOR = ':'
-
-
# HostMount()
#
# A simple object describing the behavior of
@@ -135,8 +131,8 @@ class Project():
# fully qualified urls based on the shorthand which is allowed
# to be specified in the YAML
def translate_url(self, url):
- if url and _ALIAS_SEPARATOR in url:
- url_alias, url_body = url.split(_ALIAS_SEPARATOR, 1)
+ if url and utils._ALIAS_SEPARATOR in url:
+ url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
alias_url = self._aliases.get(url_alias)
if alias_url:
url = alias_url + url_body
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 8e72196..595a040 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -43,6 +43,10 @@ from . import _signals
from ._exceptions import BstError, ErrorDomain
+# The separator we use for user specified aliases
+_ALIAS_SEPARATOR = ':'
+
+
class UtilError(BstError):
"""Raised by utility functions when system calls fail.
[buildstream] 11/16: Fix parse and store
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2cbe8e89b38a174a737e4396d50d3858c5de6d35
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Wed May 16 14:13:58 2018 +0100
Fix parse and store
---
buildstream/_project.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/buildstream/_project.py b/buildstream/_project.py
index b23a5a8..9fce29e 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -476,7 +476,6 @@ class Project():
self._shell_host_files.append(mount)
mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
- default_mirror_set = False
for mirror in mirrors:
allowed_mirror_fields = [
'location-name', 'aliases'
@@ -488,9 +487,8 @@ class Project():
assert isinstance(uris, list)
alias_mappings[alias_mapping] = list(uris)
self.mirrors[mirror_location] = alias_mappings
- if not default_mirror_set:
+ if not self.default_mirror:
self.default_mirror = mirror_location
- default_mirror_set = True
# _assert_plugin_format()
#
[buildstream] 01/16: project: Parse and store mirrors
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit bb4ace1c60a6b5b327dacc1cfbc8f0401141f4c4
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Mon Apr 9 15:48:50 2018 +0100
project: Parse and store mirrors
---
buildstream/_project.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 81 insertions(+), 2 deletions(-)
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 5344e95..d1827d7 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -20,7 +20,7 @@
import os
import multiprocessing # for cpu_count()
-from collections import Mapping
+from collections import Mapping, OrderedDict
from pluginbase import PluginBase
from . import utils
from . import _cachekey
@@ -94,6 +94,8 @@ class Project():
self.base_env_nocache = None # The base nocache mask (list) for the environment
self.element_overrides = {} # Element specific configurations
self.source_overrides = {} # Source specific configurations
+ self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs.
+ self.default_mirror = None # The name of the preferred mirror.
#
# Private Members
@@ -202,6 +204,66 @@ class Project():
self._assert_plugin_format(source, version)
return source
+ # get_alias_uri()
+ #
+ # Returns the URI for a given alias, if it exists
+ #
+ # Args:
+ # alias (str): The alias.
+ #
+ # Returns:
+ # str: The URI for the given alias; or None: if there is no URI for
+ # that alias.
+ def get_alias_uri(self, alias):
+ return self._aliases.get(alias)
+
+ # generate_alias_combinations()
+ #
+ # Yields every unique combination of mirrors for each alias
+ #
+ # e.g. alias 'foo' has a mirror at 'mirror-A', and the normal alias at 'upstream-A'
+ # alias 'bar' has no mirror, but does have the normal alias at 'upstream-B'
+ # We would yield {'foo': 'mirror-A', 'bar': 'upstream-B'},
+ # and {'foo': 'upstream-A', 'bar': 'upstream-B'},
+ #
+ # Args:
+ # aliases (list): A list of aliases to generate combinations for
+ #
+ # Yields:
+ # a dict mapping aliases to a mirror URI
+ #
+ def generate_alias_combinations(self, aliases):
+ # We numerically address the aliases
+ aliases = list(aliases)
+
+ # Flatten the mirrors and put them in the right order
+ flattened_mirrors = {}
+ for alias in aliases:
+ flattened_mirrors[alias] = []
+ for mirror_location, alias_mappings in self.mirrors.items():
+ if alias in alias_mappings:
+ mapping_list = list(alias_mappings[alias])
+ if mirror_location == self.default_mirror:
+ # The default mirror goes first
+ flattened_mirrors[alias] = mapping_list + flattened_mirrors[alias]
+ else:
+ flattened_mirrors[alias].extend(mapping_list)
+ flattened_mirrors[alias].append(self._aliases[alias])
+
+ combinations = [[]]
+ for alias in aliases:
+ new_combinations = []
+ for x in combinations:
+ for y in flattened_mirrors[alias]:
+ new_combinations.append(x + [y])
+ combinations = new_combinations
+
+ for combination in combinations:
+ out_combination = {}
+ for i, alias in enumerate(aliases):
+ out_combination[alias] = combination[i]
+ yield out_combination
+
# _load():
#
# Loads the project configuration file in the project directory.
@@ -249,7 +311,7 @@ class Project():
'aliases', 'name',
'artifacts', 'options',
'fail-on-overlap', 'shell',
- 'ref-storage', 'sandbox'
+ 'ref-storage', 'sandbox', 'mirrors',
])
# The project name, element path and option declarations
@@ -417,6 +479,23 @@ class Project():
self._shell_host_files.append(mount)
+ mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
+ default_mirror_set = False
+ for mirror in mirrors:
+ allowed_mirror_fields = [
+ 'location-name', 'aliases'
+ ]
+ _yaml.node_validate(mirror, allowed_mirror_fields)
+ mirror_location = _yaml.node_get(mirror, str, 'location-name')
+ alias_mappings = {}
+ for alias_mapping, uris in _yaml.node_items(mirror['aliases']):
+ assert isinstance(uris, list)
+ alias_mappings[alias_mapping] = list(uris)
+ self.mirrors[mirror_location] = alias_mappings
+ if not default_mirror_set:
+ self.default_mirror = mirror_location
+ default_mirror_set = True
+
# _assert_plugin_format()
#
# Helper to raise a PluginError if the loaded plugin is of a lesser version then
[buildstream] 12/16: Set default mirror via command-line or user
config
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 32b110ea055613d3048599ccd9da80032c6ba838
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Wed May 16 14:15:47 2018 +0100
Set default mirror via command-line or user config
In user config (buildstream.conf), it is set with the "default-mirror"
field.
On the command-line, it is set with "--default-mirror"
---
buildstream/_context.py | 6 ++++++
buildstream/_frontend/app.py | 3 ++-
buildstream/_frontend/cli.py | 2 ++
buildstream/_project.py | 5 +++--
tests/completions/completions.py | 1 +
5 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/buildstream/_context.py b/buildstream/_context.py
index bf7f495..5410a71 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -101,6 +101,9 @@ class Context():
# What to do when a build fails in non interactive mode
self.sched_error_action = 'continue'
+ # The default mirror to fetch from
+ self.default_mirror = None
+
# Whether elements must be rebuilt when their dependencies have changed
self._strict_build_plan = None
@@ -152,6 +155,7 @@ class Context():
_yaml.node_validate(defaults, [
'sourcedir', 'builddir', 'artifactdir', 'logdir',
'scheduler', 'artifacts', 'logging', 'projects',
+ 'default-mirror',
])
for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
@@ -196,6 +200,8 @@ class Context():
# Load per-projects overrides
self._project_overrides = _yaml.node_get(defaults, Mapping, 'projects', default_value={})
+ self.default_mirror = _yaml.node_get(defaults, str, "default-mirror", default_value=None)
+
# Shallow validation of overrides, parts of buildstream which rely
# on the overrides are expected to validate elsewhere.
for _, overrides in _yaml.node_items(self._project_overrides):
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index fa07a9a..2ab9106 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -204,7 +204,8 @@ class App():
# Load the Project
#
try:
- self.project = Project(directory, self.context, cli_options=self._main_options['option'])
+ self.project = Project(directory, self.context, cli_options=self._main_options['option'],
+ default_mirror=self._main_options.get('default_mirror'))
except LoadError as e:
# Let's automatically start a `bst init` session in this case
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index c321fa9..b2308f9 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -159,6 +159,8 @@ def print_version(ctx, param, value):
help="Elements must be rebuilt when their dependencies have changed")
@click.option('--option', '-o', type=click.Tuple([str, str]), multiple=True, metavar='OPTION VALUE',
help="Specify a project option")
+@click.option('--default-mirror', default=None,
+ help="The mirror to fetch from first, before attempting other mirrors")
@click.pass_context
def cli(context, **kwargs):
"""Build and manipulate BuildStream projects
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 9fce29e..90a73d5 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -65,7 +65,7 @@ class HostMount():
#
class Project():
- def __init__(self, directory, context, *, junction=None, cli_options=None):
+ def __init__(self, directory, context, *, junction=None, cli_options=None, default_mirror=None):
# The project name
self.name = None
@@ -91,7 +91,8 @@ class Project():
self.element_overrides = {} # Element specific configurations
self.source_overrides = {} # Source specific configurations
self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs.
- self.default_mirror = None # The name of the preferred mirror.
+
+ self.default_mirror = default_mirror or context.default_mirror # The name of the preferred mirror.
#
# Private Members
diff --git a/tests/completions/completions.py b/tests/completions/completions.py
index cc98cb9..ec6ae9a 100644
--- a/tests/completions/completions.py
+++ b/tests/completions/completions.py
@@ -26,6 +26,7 @@ MAIN_OPTIONS = [
"--colors ",
"--config ",
"--debug ",
+ "--default-mirror ",
"--directory ",
"--error-lines ",
"--fetchers ",
[buildstream] 05/16: testutils: Add a helper to copy a testutils
repo
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c5db5de7af054571efd0ea13800fb7ea4eafea6d
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Fri Apr 13 16:44:13 2018 +0100
testutils: Add a helper to copy a testutils repo
This is helpful if you want to test what happens when you have one repo
that has diverged from another. By copying the repo you're sure they
start with shared history.
This is especially useful when mirroring.
---
tests/testutils/repo/repo.py | 21 ++++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
diff --git a/tests/testutils/repo/repo.py b/tests/testutils/repo/repo.py
index 4c9ee59..61dee7b 100644
--- a/tests/testutils/repo/repo.py
+++ b/tests/testutils/repo/repo.py
@@ -22,7 +22,7 @@ class Repo():
# The directory the actual repo will be stored in
self.repo = os.path.join(self.directory, subdir)
- os.makedirs(self.repo)
+ os.makedirs(self.repo, exist_ok=True)
# create():
#
@@ -69,3 +69,22 @@ class Repo():
shutil.copytree(src_path, dest_path)
else:
shutil.copy2(src_path, dest_path)
+
+ # copy():
+ #
+ # Creates a copy of this repository in the specified
+ # destination.
+ #
+ # Args:
+ # dest (str): The destination directory
+ #
+ # Returns:
+ # (Repo): A Repo object for the new repository.
+ def copy(self, dest):
+ subdir = os.path.basename(self.repo)
+ new_dir = os.path.join(dest, subdir)
+ os.makedirs(new_dir, exist_ok=True)
+ self.copy_directory(self.repo, new_dir)
+ repo_type = type(self)
+ new_repo = repo_type(dest, subdir)
+ return new_repo
[buildstream] 04/16: bzr.py: Improve mirror support
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 463199d0cceb41c5943992f4affc063f1cc9aae2
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Fri Apr 13 16:47:18 2018 +0100
bzr.py: Improve mirror support
This fixes:
* Bzr repositories pulling from the branch they were created with.
* Bzr's _ensure_mirror() not actually checking that it successfully
mirrored the ref.
---
buildstream/plugins/sources/bzr.py | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/buildstream/plugins/sources/bzr.py b/buildstream/plugins/sources/bzr.py
index 3732304..7e784ff 100644
--- a/buildstream/plugins/sources/bzr.py
+++ b/buildstream/plugins/sources/bzr.py
@@ -97,7 +97,7 @@ class BzrSource(Source):
def track(self):
with self.timed_activity("Tracking {}".format(self.url),
silent_nested=True):
- self._ensure_mirror()
+ self._ensure_mirror(skip_ref_check=True)
ret, out = self.check_output([self.host_bzr, "version-info",
"--custom", "--template={revno}",
self._get_branch_dir()],
@@ -207,7 +207,7 @@ class BzrSource(Source):
yield repodir
self._atomic_replace_mirrordir(repodir)
- def _ensure_mirror(self):
+ def _ensure_mirror(self, skip_ref_check=False):
with self._atomic_repodir() as repodir:
# Initialize repo if no metadata
bzr_metadata_dir = os.path.join(repodir, ".bzr")
@@ -216,18 +216,21 @@ class BzrSource(Source):
fail="Failed to initialize bzr repository")
branch_dir = os.path.join(repodir, self.tracking)
+ branch_url = self.url + "/" + self.tracking
if not os.path.exists(branch_dir):
# `bzr branch` the branch if it doesn't exist
# to get the upstream code
- branch_url = self.url + "/" + self.tracking
self.call([self.host_bzr, "branch", branch_url, branch_dir],
fail="Failed to branch from {} to {}".format(branch_url, branch_dir))
else:
# `bzr pull` the branch if it does exist
# to get any changes to the upstream code
- self.call([self.host_bzr, "pull", "--directory={}".format(branch_dir)],
+ self.call([self.host_bzr, "pull", "--directory={}".format(branch_dir), branch_url],
fail="Failed to pull new changes for {}".format(branch_dir))
+ if not skip_ref_check and not self._check_ref():
+ raise SourceError("Failed to ensure ref '{}' was mirrored".format(self.ref),
+ reason="ref-not-mirrored")
def setup():
[buildstream] 07/16: git.py: Improve mirror support
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 277cae92c0a84fa64fc1b01022f1a8e34ace5890
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Thu Apr 12 16:28:12 2018 +0100
git.py: Improve mirror support
When implementing fetching from mirrors, I encountered some problems
with the git source:
1. The mirror URL was using translate_url()'s output, so if a different
alias was used, then fetching from the mirror would go to a different
directory, and be inaccessible.
2. After fixing that, fetching was unable to pull from a URL other than
the one used at repository creation, meaning it wouldn't actually
pull from the mirror.
---
buildstream/plugins/sources/git.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index f178656..20393d3 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -91,7 +91,7 @@ class GitMirror():
self.path = path
self.url = source.translate_url(url)
self.ref = ref
- self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(self.url))
+ self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
# Ensures that the mirror exists
def ensure(self):
@@ -117,7 +117,7 @@ class GitMirror():
.format(self.source, self.url, tmpdir, self.mirror)) from e
def fetch(self):
- self.source.call([self.source.host_git, 'fetch', 'origin', '--prune'],
+ self.source.call([self.source.host_git, 'fetch', self.url, '--prune'],
fail="Failed to fetch from remote git repository: {}".format(self.url),
cwd=self.mirror)