You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:34:42 UTC
[buildstream] branch 537-mirror-fallback-does-not-work-for-git
created (now cc1202d)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch 537-mirror-fallback-does-not-work-for-git
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at cc1202d Attempt to fix further problems with the git source
This branch includes the following new commits:
new b22ce3c tests: Test that fetching passes when upstream is absent
new f969d81 source.py: Fix documentation on get_source_fetchers
new c3befa7 git.py: Provide a list of source fetchers without downloading anything
new a80bd2d Exclude ostree from tests on fetching with missing upstream
new 6ef51eb tests: Add a test of the git source with submodules
new cc1202d Attempt to fix further problems with the git source
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 06/06: Attempt to fix further problems with the git
source
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch 537-mirror-fallback-does-not-work-for-git
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit cc1202d5adf4d40c7434d1b4ce932d4093fd82ad
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Wed Aug 8 18:24:18 2018 +0100
Attempt to fix further problems with the git source
---
buildstream/plugins/sources/git.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index 802d4ff..ce7985f 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -102,6 +102,7 @@ class GitMirror(SourceFetcher):
self._path = path
self._ref = ref
+ self._alias_override = None
# Ensures that the mirror exists
def ensure(self, alias_override=None):
@@ -163,6 +164,7 @@ class GitMirror(SourceFetcher):
cwd=self.mirror)
def fetch(self, alias_override=None):
+ self._alias_override = alias_override
self.ensure(alias_override)
if not self.has_ref():
self._fetch(alias_override)
@@ -294,12 +296,13 @@ class GitMirror(SourceFetcher):
return None
def get_submodule_path(self, url):
+ real_url = self.source.translate_url(url)
for parser, section in self._read_gitmodules():
parsed_url = parser.get(section, 'url')
- if parsed_url == url:
+ if parsed_url == real_url:
return parser.get(section, 'path')
- raise SourceError("{}: No submodule found with url '{}'".format(self.source, url))
+ raise SourceError("{}: No submodule found with url '{}'".format(self.source, real_url))
@property
def path(self):
[buildstream] 04/06: Exclude ostree from tests on fetching with
missing upstream
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch 537-mirror-fallback-does-not-work-for-git
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit a80bd2dc6d7968d2685405c2fee8a40cd0467c70
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Wed Aug 8 10:44:01 2018 +0100
Exclude ostree from tests on fetching with missing upstream
OSTree sources currently fail and there is an issue to fix it, but this
branch can't be merged when tests fail.
---
tests/frontend/mirror.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
index f658283..bcc4e1b 100644
--- a/tests/frontend/mirror.py
+++ b/tests/frontend/mirror.py
@@ -140,7 +140,7 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS if kind is not "ostree"])
def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
[buildstream] 01/06: tests: Test that fetching passes when upstream
is absent
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch 537-mirror-fallback-does-not-work-for-git
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b22ce3cb7b000702a10fa4c817851ab0afbe3bf0
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Mon Aug 6 13:31:50 2018 +0100
tests: Test that fetching passes when upstream is absent
---
tests/frontend/mirror.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 57 insertions(+)
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
index f37cc18..f658283 100644
--- a/tests/frontend/mirror.py
+++ b/tests/frontend/mirror.py
@@ -140,6 +140,63 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
+ bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+ dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+ upstream_repodir = os.path.join(str(tmpdir), 'upstream')
+ mirror_repodir = os.path.join(str(tmpdir), 'mirror')
+ project_dir = os.path.join(str(tmpdir), 'project')
+ os.makedirs(project_dir)
+ element_dir = os.path.join(project_dir, 'elements')
+
+ # Create repo objects of the upstream and mirror
+ upstream_repo = create_repo(kind, upstream_repodir)
+ ref = upstream_repo.create(dev_files_path)
+ mirror_repo = upstream_repo.copy(mirror_repodir)
+
+ element = {
+ 'kind': 'import',
+ 'sources': [
+ upstream_repo.source_config(ref=ref)
+ ]
+ }
+
+ element_name = 'test.bst'
+ element_path = os.path.join(element_dir, element_name)
+ full_repo = element['sources'][0]['url']
+ upstream_map, repo_name = os.path.split(full_repo)
+ alias = 'foo-' + kind
+ aliased_repo = alias + ':' + repo_name
+ element['sources'][0]['url'] = aliased_repo
+ full_mirror = mirror_repo.source_config()['url']
+ mirror_map, _ = os.path.split(full_mirror)
+ os.makedirs(element_dir)
+ _yaml.dump(element, element_path)
+
+ project = {
+ 'name': 'test',
+ 'element-path': 'elements',
+ 'aliases': {
+ alias: 'http://www.example.com/'
+ },
+ 'mirrors': [
+ {
+ 'name': 'middle-earth',
+ 'aliases': {
+ alias: [mirror_map + "/"],
+ },
+ },
+ ]
+ }
+ project_file = os.path.join(project_dir, 'project.conf')
+ _yaml.dump(project, project_file)
+
+ result = cli.run(project=project_dir, args=['fetch', element_name])
+ result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
def test_mirror_fetch_multi(cli, tmpdir, datafiles):
output_file = os.path.join(str(tmpdir), "output.txt")
project_dir = str(tmpdir)
[buildstream] 05/06: tests: Add a test of the git source with
submodules
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch 537-mirror-fallback-does-not-work-for-git
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6ef51eb61daf2447f4039ecc35b5f86a67028a47
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Wed Aug 8 18:23:52 2018 +0100
tests: Add a test of the git source with submodules
---
tests/frontend/mirror.py | 79 ++++++++++++++++++++++++++++++++++++++++
tests/frontend/project/files/bar | 0
tests/frontend/project/files/foo | 0
3 files changed, 79 insertions(+)
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
index bcc4e1b..7a664d1 100644
--- a/tests/frontend/mirror.py
+++ b/tests/frontend/mirror.py
@@ -616,3 +616,82 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
os.rename('{}.bak'.format(upstream_repo.repo), upstream_repo.repo)
result = cli.run(project=project_dir, args=['fetch', element_name])
result.assert_success()
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles):
+ # Test that it behaves as expected with submodules, both defined in config
+ # and discovered when fetching.
+ foo_file = os.path.join(str(datafiles), 'files', 'foo')
+ bar_file = os.path.join(str(datafiles), 'files', 'bar')
+ bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+ dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+ mirror_dir = os.path.join(str(datafiles), 'mirror')
+
+ defined_subrepo = create_repo('git', str(tmpdir), 'defined_subrepo')
+ defined_mirror_ref = defined_subrepo.create(bin_files_path)
+ defined_mirror = defined_subrepo.copy(mirror_dir)
+ defined_subref = defined_subrepo.add_file(foo_file)
+
+ found_subrepo = create_repo('git', str(tmpdir), 'found_subrepo')
+ found_subref = found_subrepo.create(dev_files_path)
+
+ main_repo = create_repo('git', str(tmpdir))
+ main_mirror_ref = main_repo.create(bin_files_path)
+ main_repo.add_submodule('defined', 'file://' + defined_subrepo.repo)
+ main_repo.add_submodule('found', 'file://' + found_subrepo.repo)
+ main_mirror = main_repo.copy(mirror_dir)
+ main_ref = main_repo.add_file(bar_file)
+
+ project_dir = os.path.join(str(tmpdir), 'project')
+ os.makedirs(project_dir)
+ element_dir = os.path.join(project_dir, 'elements')
+ os.makedirs(element_dir)
+ element = {
+ 'kind': 'import',
+ 'sources': [
+ main_repo.source_config(ref=main_mirror_ref)
+ ]
+ }
+ element_name = 'test.bst'
+ element_path = os.path.join(element_dir, element_name)
+
+ # Alias the main repo
+ full_repo = element['sources'][0]['url']
+ _, repo_name = os.path.split(full_repo)
+ alias = 'foo'
+ aliased_repo = alias + ':' + repo_name
+ element['sources'][0]['url'] = aliased_repo
+
+ # Hide the found subrepo
+ del element['sources'][0]['submodules']['found']
+
+ # Alias the defined subrepo
+ subrepo = element['sources'][0]['submodules']['defined']['url']
+ _, repo_name = os.path.split(subrepo)
+ aliased_repo = alias + ':' + repo_name
+ element['sources'][0]['submodules']['defined']['url'] = aliased_repo
+
+ _yaml.dump(element, element_path)
+
+ full_mirror = main_mirror.source_config()['url']
+ mirror_map, _ = os.path.split(full_mirror)
+ project = {
+ 'name': 'test',
+ 'element-path': 'elements',
+ 'aliases': {
+ alias: 'http://www.example.com/'
+ },
+ 'mirrors': [
+ {
+ 'name': 'middle-earth',
+ 'aliases': {
+ alias: [mirror_map + "/"],
+ },
+ },
+ ]
+ }
+ project_file = os.path.join(project_dir, 'project.conf')
+ _yaml.dump(project, project_file)
+
+ result = cli.run(project=project_dir, args=['fetch', element_name])
+ result.assert_success()
diff --git a/tests/frontend/project/files/bar b/tests/frontend/project/files/bar
new file mode 100644
index 0000000..e69de29
diff --git a/tests/frontend/project/files/foo b/tests/frontend/project/files/foo
new file mode 100644
index 0000000..e69de29
[buildstream] 03/06: git.py: Provide a list of source fetchers
without downloading anything
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch 537-mirror-fallback-does-not-work-for-git
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c3befa7b2751af05e8a6ea45f8e13a300e73a15e
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Mon Aug 6 19:28:21 2018 +0100
git.py: Provide a list of source fetchers without downloading anything
---
buildstream/plugins/sources/git.py | 139 ++++++++++++++++++++++++++-----------
1 file changed, 97 insertions(+), 42 deletions(-)
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index a7d3ad6..802d4ff 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -91,16 +91,18 @@ GIT_MODULES = '.gitmodules'
#
class GitMirror(SourceFetcher):
- def __init__(self, source, path, url, ref):
+ def __init__(self, source, path, url, ref, *, parent=None):
super().__init__()
self.source = source
- self.path = path
+ self.parent = parent
self.url = url
- self.ref = ref
self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
self.mark_download_url(url)
+ self._path = path
+ self._ref = ref
+
# Ensures that the mirror exists
def ensure(self, alias_override=None):
@@ -223,8 +225,7 @@ class GitMirror(SourceFetcher):
fail="Failed to checkout git ref {}".format(self.ref),
cwd=fullpath)
- # List the submodules (path/url tuples) present at the given ref of this repo
- def submodule_list(self):
+ def _read_gitmodules(self):
modules = "{}:{}".format(self.ref, GIT_MODULES)
exit_code, output = self.source.check_output(
[self.source.host_git, 'show', modules], cwd=self.mirror)
@@ -247,10 +248,15 @@ class GitMirror(SourceFetcher):
for section in parser.sections():
# validate section name against the 'submodule "foo"' pattern
if re.match(r'submodule "(.*)"', section):
- path = parser.get(section, 'path')
- url = parser.get(section, 'url')
+ yield (parser, section)
- yield (path, url)
+ # List the submodules (path/url tuples) present at the given ref of this repo
+ def submodule_list(self):
+ for parser, section in self._read_gitmodules():
+ path = parser.get(section, 'path')
+ url = parser.get(section, 'url')
+
+ yield (path, url)
# Fetch the ref which this mirror requires its submodule to have,
# at the given ref of this mirror.
@@ -287,6 +293,76 @@ class GitMirror(SourceFetcher):
return None
+ def get_submodule_path(self, url):
+ for parser, section in self._read_gitmodules():
+ parsed_url = parser.get(section, 'url')
+ if parsed_url == url:
+ return parser.get(section, 'path')
+
+ raise SourceError("{}: No submodule found with url '{}'".format(self.source, url))
+
+ @property
+ def path(self):
+ if self._path is None:
+ self._path = self.parent.get_submodule_path(self.url)
+
+ return self._path
+
+ @property
+ def ref(self):
+ # The top-level GitMirror may have ref as None, submodules don't.
+ if self._ref is None and self.parent:
+ self._ref = self.parent.submodule_ref(self.path)
+
+ return self._ref
+
+ @ref.setter
+ def ref(self, ref):
+ self._ref = ref
+
+
+# A SourceFetcher that may also check for, and have submodules of its own.
+class TopLevelGitMirror(GitMirror):
+ def __init__(self, source, path, url, ref):
+ super().__init__(source, path, url, ref)
+ self.auto_submodules = []
+
+ def fetch(self, alias_override=None):
+ super().fetch(alias_override)
+ self.refresh_submodules()
+
+ # auto_submodules do not have aliases, so don't need an override
+ for mirror in self.auto_submodules:
+ mirror.fetch()
+
+ # Refreshes the GitMirror objects for submodules
+ #
+ # Assumes that we have our mirror and we have the ref which we point to
+ #
+ def refresh_submodules(self):
+ self.ensure()
+
+ excluded_paths = list([s.path for s in self.source.manual_submodules])
+ submodules = []
+
+ # XXX Here we should issue a warning if either:
+ # A.) A submodule exists but is not defined in the element configuration
+ # B.) The element configuration configures submodules which dont exist at the current ref
+ #
+ for path, url in self.submodule_list():
+ if path in excluded_paths:
+ continue
+ else:
+ self.source.warn("Unexpected submodule detected with path '{}' and url '{}'"
+ .format(path, url))
+
+ ref = self.submodule_ref(path)
+ if ref is not None:
+ mirror = GitMirror(self, path, url, ref)
+ submodules.append(mirror)
+
+ self.auto_submodules = submodules
+
class GitSource(Source):
# pylint: disable=attribute-defined-outside-init
@@ -298,10 +374,10 @@ class GitSource(Source):
self.node_validate(node, config_keys + Source.COMMON_CONFIG_KEYS)
self.original_url = self.node_get_member(node, str, 'url')
- self.mirror = GitMirror(self, '', self.original_url, ref)
+ self.mirror = TopLevelGitMirror(self, '', self.original_url, ref)
self.tracking = self.node_get_member(node, str, 'track', None)
self.checkout_submodules = self.node_get_member(node, bool, 'checkout-submodules', True)
- self.submodules = []
+ self.manual_submodules = []
# Parse a dict of submodule overrides, stored in the submodule_overrides
# and submodule_checkout_overrides dictionaries.
@@ -311,6 +387,9 @@ class GitSource(Source):
for path, _ in self.node_items(modules):
submodule = self.node_get_member(modules, Mapping, path)
url = self.node_get_member(submodule, str, 'url', None)
+ submodule_mirror = GitMirror(self, None, url, None, parent=self.mirror)
+ self.manual_submodules.append(submodule_mirror)
+
self.submodule_overrides[path] = url
if 'checkout' in submodule:
checkout = self.node_get_member(submodule, bool, 'checkout')
@@ -384,7 +463,7 @@ class GitSource(Source):
def init_workspace(self, directory):
# XXX: may wish to refactor this as some code dupe with stage()
- self.refresh_submodules()
+ self.mirror.refresh_submodules()
with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
self.mirror.init_workspace(directory)
@@ -398,7 +477,7 @@ class GitSource(Source):
# with submodules present (source needed fetching) and
# we may not know about the submodule yet come time to build.
#
- self.refresh_submodules()
+ self.mirror.refresh_submodules()
# Stage the main repo in the specified directory
#
@@ -414,17 +493,20 @@ class GitSource(Source):
mirror.stage(directory)
def get_source_fetchers(self):
- self.refresh_submodules()
- return [self.mirror] + self.submodules
+ return [self.mirror] + self.manual_submodules
###########################################################
# Local Functions #
###########################################################
+ @property
+ def submodules(self):
+ return self.manual_submodules + self.mirror.auto_submodules
+
def have_all_refs(self):
if not self.mirror.has_ref():
return False
- self.refresh_submodules()
+ self.mirror.refresh_submodules()
for mirror in self.submodules:
if not os.path.exists(mirror.mirror):
return False
@@ -433,33 +515,6 @@ class GitSource(Source):
return True
- # Refreshes the GitMirror objects for submodules
- #
- # Assumes that we have our mirror and we have the ref which we point to
- #
- def refresh_submodules(self):
- self.mirror.ensure()
- submodules = []
-
- # XXX Here we should issue a warning if either:
- # A.) A submodule exists but is not defined in the element configuration
- # B.) The element configuration configures submodules which dont exist at the current ref
- #
- for path, url in self.mirror.submodule_list():
-
- # Allow configuration to override the upstream
- # location of the submodules.
- override_url = self.submodule_overrides.get(path)
- if override_url:
- url = override_url
-
- ref = self.mirror.submodule_ref(path)
- if ref is not None:
- mirror = GitMirror(self, path, url, ref)
- submodules.append(mirror)
-
- self.submodules = submodules
-
# Plugin entry point
def setup():
[buildstream] 02/06: source.py: Fix documentation on
get_source_fetchers
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch 537-mirror-fallback-does-not-work-for-git
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f969d8108630c3fda5a39f6791c408fd49a45a01
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Mon Aug 6 19:26:13 2018 +0100
source.py: Fix documentation on get_source_fetchers
---
buildstream/source.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/buildstream/source.py b/buildstream/source.py
index d58bfe2..9343f08 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -393,8 +393,8 @@ class Source(Plugin):
"""Get the objects that are used for fetching
If this source doesn't download from multiple URLs,
- returning None and falling back on the default behaviour
- is recommended.
+ returning an empty list and falling back on the default
+ behaviour is recommended.
Returns:
list: A list of SourceFetchers. If SourceFetchers are not supported,