You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:39:38 UTC
[buildstream] 12/18: git.py: Use SourceDownloaders and
alias_overrides
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jonathan/mirror-client-sourcedownloader
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e24bcd9438c37c179518de71ee47513ca20720c9
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Thu Apr 12 16:28:12 2018 +0100
git.py: Use SourceDownloaders and alias_overrides
Each GitMirror is now a SourceDownloader
---
buildstream/plugins/sources/git.py | 123 ++++++++++++++++++-------------------
1 file changed, 59 insertions(+), 64 deletions(-)
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index d079d87..63d528d 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -78,7 +78,7 @@ from io import StringIO
from configparser import RawConfigParser
-from buildstream import Source, SourceError, Consistency
+from buildstream import Source, SourceError, Consistency, SourceDownloader
from buildstream import utils
GIT_MODULES = '.gitmodules'
@@ -88,18 +88,19 @@ GIT_MODULES = '.gitmodules'
# for the primary git source and also for each submodule it
# might have at a given time
#
-class GitMirror():
+class GitMirror(SourceDownloader):
def __init__(self, source, path, url, ref):
self.source = source
self.path = path
- self.url = source.translate_url(url)
+ self.original_url = url
self.ref = ref
- self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(self.url))
+ self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
# Ensures that the mirror exists
- def ensure(self):
+ def ensure(self, alias_override=None):
+ url = self.source.translate_url(self.original_url, alias_override)
# Unfortunately, git does not know how to only clone just a specific ref,
# so we have to download all of those gigs even if we only need a couple
@@ -112,19 +113,47 @@ class GitMirror():
# system configured tmpdir is not on the same partition.
#
with self.source.tempdir() as tmpdir:
- self.source.call([self.source.host_git, 'clone', '--mirror', '-n', self.url, tmpdir],
- fail="Failed to clone git repository {}".format(self.url))
+ self.source.call([self.source.host_git, 'clone', '--mirror', '-n', url, tmpdir],
+ fail="Failed to clone git repository {}".format(url))
try:
shutil.move(tmpdir, self.mirror)
except (shutil.Error, OSError) as e:
raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}'"
- .format(self.source, self.url, tmpdir, self.mirror)) from e
+ .format(self.source, url, tmpdir, self.mirror)) from e
+
+ def fetch(self, alias_override=None):
+ url = self.source.translate_url(self.original_url)
+ with self.source.timed_activity("Fetching {}".format(url), silent_nested=True):
+ self.ensure(alias_override)
+ # NOTE: We fetch from the local source cache if possible, potentially causing
+ # unexpected results when fetching from mirrors
+ if not self.has_ref():
+ self.source.call([self.source.host_git, 'fetch', url, '--prune'],
+ fail="Failed to fetch from remote git repository: {}".format(url),
+ cwd=self.mirror)
+ self.assert_ref()
+
+ def track(self, alias_override=None):
+ url = self.source.translate_url(self.original_url, alias_override)
- def fetch(self):
- self.source.call([self.source.host_git, 'fetch', 'origin', '--prune'],
- fail="Failed to fetch from remote git repository: {}".format(self.url),
- cwd=self.mirror)
+ # If self.tracking is not specified it's not an error, just silently return
+ if not self.source.tracking:
+ return None
+
+ with self.source.timed_activity("Tracking {} from {}"
+ .format(self.source.tracking, url),
+ silent_nested=True):
+ self.ensure(alias_override=alias_override)
+ if not self.has_ref():
+ self.source.call([self.source.host_git, 'fetch', url, '--prune'],
+ fail="Failed to fetch from remote git repository: {}".format(url),
+ cwd=self.mirror)
+
+ # Update self.ref and node.ref from the self.tracking branch
+ ret = self.latest_commit(self.source.tracking)
+
+ return ret
def has_ref(self):
if not self.ref:
@@ -141,7 +170,7 @@ class GitMirror():
def assert_ref(self):
if not self.has_ref():
raise SourceError("{}: expected ref '{}' was not found in git repository: '{}'"
- .format(self.source, self.ref, self.url))
+ .format(self.source, self.ref, self.original_url))
def latest_commit(self, tracking):
_, output = self.source.check_output(
@@ -164,13 +193,14 @@ class GitMirror():
cwd=fullpath)
def init_workspace(self, directory):
+ url = self.source.translate_url(self.original_url)
fullpath = os.path.join(directory, self.path)
self.source.call([self.source.host_git, 'clone', '--no-checkout', self.mirror, fullpath],
fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath))
- self.source.call([self.source.host_git, 'remote', 'set-url', 'origin', self.url],
- fail='Failed to add remote origin "{}"'.format(self.url),
+ self.source.call([self.source.host_git, 'remote', 'set-url', 'origin', url],
+ fail='Failed to add remote origin "{}"'.format(url),
cwd=fullpath)
self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
@@ -241,6 +271,13 @@ class GitMirror():
return None
+ def get_alias(self):
+ if utils._ALIAS_SEPARATOR in self.original_url:
+ alias, _ = self.original_url.split(utils._ALIAS_SEPARATOR, 1)
+ return alias
+ else:
+ return None
+
class GitSource(Source):
# pylint: disable=attribute-defined-outside-init
@@ -311,41 +348,6 @@ class GitSource(Source):
def set_ref(self, ref, node):
node['ref'] = self.mirror.ref = ref
- def track(self):
-
- # If self.tracking is not specified it's not an error, just silently return
- if not self.tracking:
- return None
-
- with self.timed_activity("Tracking {} from {}"
- .format(self.tracking, self.mirror.url),
- silent_nested=True):
- self.mirror.ensure()
- self.mirror.fetch()
-
- # Update self.mirror.ref and node.ref from the self.tracking branch
- ret = self.mirror.latest_commit(self.tracking)
-
- return ret
-
- def fetch(self):
-
- with self.timed_activity("Fetching {}".format(self.mirror.url), silent_nested=True):
-
- # Here we are only interested in ensuring that our mirror contains
- # the self.mirror.ref commit.
- self.mirror.ensure()
- if not self.mirror.has_ref():
- self.mirror.fetch()
-
- self.mirror.assert_ref()
-
- # Here after performing any fetches, we need to also ensure that
- # we've cached the desired refs in our mirrors of submodules.
- #
- self.refresh_submodules()
- self.fetch_submodules()
-
def init_workspace(self, directory):
# XXX: may wish to refactor this as some code dupe with stage()
self.refresh_submodules()
@@ -366,7 +368,8 @@ class GitSource(Source):
# Stage the main repo in the specified directory
#
- with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True):
+ url = self.translate_url(self.mirror.original_url)
+ with self.timed_activity("Staging {}".format(url), silent_nested=True):
self.mirror.stage(directory)
for mirror in self.submodules:
if mirror.path in self.submodule_checkout_overrides:
@@ -377,6 +380,11 @@ class GitSource(Source):
if checkout:
mirror.stage(directory)
+ def get_source_downloaders(self, alias_override=None):
+ self.mirror.ensure(alias_override=alias_override)
+ self.refresh_submodules()
+ return [self.mirror] + self.submodules
+
###########################################################
# Local Functions #
###########################################################
@@ -419,19 +427,6 @@ class GitSource(Source):
self.submodules = submodules
- # Ensures that we have mirrored git repositories for all
- # the submodules existing at the given commit of the main git source.
- #
- # Also ensure that these mirrors have the required commits
- # referred to at the given commit of the main git source.
- #
- def fetch_submodules(self):
- for mirror in self.submodules:
- mirror.ensure()
- if not mirror.has_ref():
- mirror.fetch()
- mirror.assert_ref()
-
# Plugin entry point
def setup():