You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ro...@apache.org on 2020/12/29 13:47:32 UTC
[buildstream] branch valentindavid/update_mirror created (now
061fc28)
This is an automated email from the ASF dual-hosted git repository.
root pushed a change to branch valentindavid/update_mirror
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 061fc28 buildstream/_frontend/cli.py: Add mirror command. tests/completions/completions.py: Add mirror command.
This branch includes the following new commits:
new f539ccb buildstream/_loader/loadelement.py: Fix crash with some elements that do not have depends
new 8a51f1a buildstream/source.py: Add update_mirror() method and utilities to have namespaced mirrors
new 95fb47d buildstream/plugins/sources/_downloadablefilesource.py: Implement update_mirror.
new ff6292e buildstream/plugins/sources/git.py: Implement update_mirror.
new 127087d buildstream/plugins/sources/ostree.py: Implement update_mirror.
new 45c1161 buildstream/plugins/sources/local.py: Implement update_mirror (no-op).
new 0e483cc buildstream/plugins/sources/patch.py: Implement update_mirror (no-op).
new 8e3b236 buildstream/plugins/sources/bzr.py: Implement update_mirror.
new e0ba2b9 Add mirror tasks to stream
new 061fc28 buildstream/_frontend/cli.py: Add mirror command. tests/completions/completions.py: Add mirror command.
The 10 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 08/10: buildstream/plugins/sources/bzr.py: Implement
update_mirror.
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch valentindavid/update_mirror
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8e3b23682466fc3fcd69f242e4326435c68a28ef
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Apr 26 10:13:00 2018 +0200
buildstream/plugins/sources/bzr.py: Implement update_mirror.
---
buildstream/plugins/sources/bzr.py | 80 ++++++++++++++++++++++++--------------
1 file changed, 50 insertions(+), 30 deletions(-)
diff --git a/buildstream/plugins/sources/bzr.py b/buildstream/plugins/sources/bzr.py
index 3732304..a6171d1 100644
--- a/buildstream/plugins/sources/bzr.py
+++ b/buildstream/plugins/sources/bzr.py
@@ -113,6 +113,9 @@ class BzrSource(Source):
silent_nested=True):
self._ensure_mirror()
+ def update_mirror(self):
+ self._ensure_mirror()
+
def stage(self, directory):
self.call([self.host_bzr, "checkout", "--lightweight",
"--revision=revno:{}".format(self.ref),
@@ -137,46 +140,64 @@ class BzrSource(Source):
def _check_ref(self):
# If the mirror doesnt exist yet, then we dont have the ref
- if not os.path.exists(self._get_branch_dir()):
+ branch_dir = self._get_branch_dir()
+ if branch_dir is None:
return False
return self.call([self.host_bzr, "revno",
"--revision=revno:{}".format(self.ref),
- self._get_branch_dir()]) == 0
-
- def _get_branch_dir(self):
- return os.path.join(self._get_mirror_dir(), self.tracking)
-
- def _get_mirror_dir(self):
- return os.path.join(self.get_mirror_directory(),
- utils.url_directory_name(self.original_url))
-
- def _atomic_replace_mirrordir(self, srcdir):
+ branch_dir]) == 0
+
+ def _has_not_diverged(self, mirror_dir):
+ branch_dir = os.path.join(mirror_dir, self.tracking)
+ bzr_metadata_dir = os.path.join(branch_dir, ".bzr")
+ if not os.path.isdir(bzr_metadata_dir):
+ return True
+ branch_url = self.url + "/" + self.tracking
+ ret = self.call([self.host_bzr, "missing",
+ "--directory={}".format(branch_dir),
+ "--mine-only", "--line", "--quiet",
+ branch_url])
+ return ret == 0
+
+ def _get_branch_dir(self, create=False):
+ mirror_dir = self._get_mirror_dir(create=create)
+ if mirror_dir is None:
+ return None
+ branch_dir = os.path.join(mirror_dir, self.tracking)
+ if create:
+ os.makedirs(branch_dir, exist_ok=True)
+ return branch_dir
+
+ def _get_mirror_dir(self, create=False):
+ return self.find_mirror_directory(self.original_url, self._has_not_diverged, create=create)
+
+ def _atomic_replace_mirrordir(self, srcdir, tgtdir):
"""Helper function to safely replace the mirror dir"""
- if not os.path.exists(self._get_mirror_dir()):
+ if not os.path.exists(tgtdir):
# Just move the srcdir to the mirror dir
try:
- os.rename(srcdir, self._get_mirror_dir())
+ os.rename(srcdir, tgtdir)
except OSError as e:
raise SourceError("{}: Failed to move srcdir '{}' to mirror dir '{}'"
- .format(str(self), srcdir, self._get_mirror_dir())) from e
+ .format(str(self), srcdir, tgtdir)) from e
else:
# Atomically swap the backup dir.
- backupdir = self._get_mirror_dir() + ".bak"
+ backupdir = tgtdir + ".bak"
try:
- os.rename(self._get_mirror_dir(), backupdir)
+ os.rename(tgtdir, backupdir)
except OSError as e:
raise SourceError("{}: Failed to move mirrordir '{}' to backup dir '{}'"
- .format(str(self), self._get_mirror_dir(), backupdir)) from e
+ .format(str(self), tgtdir, backupdir)) from e
try:
- os.rename(srcdir, self._get_mirror_dir())
+ os.rename(srcdir, tgtdir)
except OSError as e:
# Attempt to put the backup back!
- os.rename(backupdir, self._get_mirror_dir())
+ os.rename(backupdir, tgtdir)
raise SourceError("{}: Failed to replace bzr repo '{}' with '{}"
- .format(str(self), srcdir, self._get_mirror_dir())) from e
+ .format(str(self), srcdir, tgtdir)) from e
finally:
if os.path.exists(backupdir):
shutil.rmtree(backupdir)
@@ -194,18 +215,17 @@ class BzrSource(Source):
repo in an inconsistent state.
"""
with self.tempdir() as repodir:
- mirror_dir = self._get_mirror_dir()
- if os.path.exists(mirror_dir):
- try:
- # shutil.copytree doesn't like it if destination exists
- shutil.rmtree(repodir)
- shutil.copytree(mirror_dir, repodir)
- except (shutil.Error, OSError) as e:
- raise SourceError("{}: Failed to copy bzr repo from '{}' to '{}'"
- .format(str(self), mirror_dir, repodir)) from e
+ mirror_dir = self._get_mirror_dir(create=True)
+ try:
+ # shutil.copytree doesn't like it if destination exists
+ shutil.rmtree(repodir)
+ shutil.copytree(mirror_dir, repodir)
+ except (shutil.Error, OSError) as e:
+ raise SourceError("{}: Failed to copy bzr repo from '{}' to '{}'"
+ .format(str(self), mirror_dir, repodir)) from e
yield repodir
- self._atomic_replace_mirrordir(repodir)
+ self._atomic_replace_mirrordir(repodir, mirror_dir)
def _ensure_mirror(self):
with self._atomic_repodir() as repodir:
[buildstream] 10/10: buildstream/_frontend/cli.py: Add mirror
command. tests/completions/completions.py: Add mirror command.
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch valentindavid/update_mirror
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 061fc2864181ed13e16a27c0370bf3cb99190c98
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Apr 26 10:14:53 2018 +0200
buildstream/_frontend/cli.py: Add mirror command.
tests/completions/completions.py: Add mirror command.
---
buildstream/_frontend/cli.py | 48 ++++++++++++++++++++++++++++++++++++++++
tests/completions/completions.py | 1 +
2 files changed, 49 insertions(+)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 41e97cb..73c94ba 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -6,6 +6,7 @@ from .. import _yaml
from .._exceptions import BstError, LoadError, AppError
from .._versions import BST_FORMAT_VERSION
from .complete import main_bashcomplete, complete_path, CompleteUnhandled
+from .. import utils
##################################################################
@@ -330,6 +331,53 @@ def track(app, elements, deps, except_, cross_junctions):
##################################################################
+# Mirror Command #
+##################################################################
+@cli.command(short_help="Mirror new source references")
+@click.option('--except', 'except_', multiple=True,
+ type=click.Path(dir_okay=False, readable=True),
+ help="Except certain dependencies from mirroring")
+@click.option('--deps', '-d', default='none',
+ type=click.Choice(['none', 'all']),
+ help='The dependencies to mirror (default: none)')
+@click.argument('elements', nargs=-1,
+ required=False,
+ type=click.Path(dir_okay=False, readable=True))
+@click.pass_obj
+def mirror(app, elements, deps, except_):
+ """Mirror all sources.
+
+ This will download everything available from sources. For
+ repositories, it will download all commits/revisions. For files,
+ it will download the latest version even if the source is already
+ cached. Previously mirrored/fetched downloads will still be
+ accessible.
+
+ If no element is given as parameters, all elements found in
+ `element_path` will be used.
+
+ Specify `--deps` to control which sources to track:
+
+ \b
+ none: No dependencies, just the specified elements
+ all: All dependencies of all specified elements
+
+ """
+ if not elements:
+ with app.partially_initialized():
+ elements = []
+ for element in utils.list_relative_paths(app.project.element_path):
+ if element.endswith('.bst'):
+ elements.append(element)
+ elements = tuple(elements)
+
+ with app.initialized(session_name="Mirror"):
+ app.stream.mirror(elements,
+ selection=deps,
+ except_targets=except_)
+
+
+##################################################################
# Pull Command #
##################################################################
@cli.command(short_help="Pull a built artifact")
diff --git a/tests/completions/completions.py b/tests/completions/completions.py
index cc98cb9..47b13dc 100644
--- a/tests/completions/completions.py
+++ b/tests/completions/completions.py
@@ -10,6 +10,7 @@ MAIN_COMMANDS = [
'checkout ',
'fetch ',
'init ',
+ 'mirror ',
'pull ',
'push ',
'shell ',
[buildstream] 05/10: buildstream/plugins/sources/ostree.py:
Implement update_mirror.
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch valentindavid/update_mirror
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 127087dcb396e56e6b165fdc1751e35d0453a71d
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Apr 26 10:08:41 2018 +0200
buildstream/plugins/sources/ostree.py: Implement update_mirror.
---
buildstream/plugins/sources/ostree.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/buildstream/plugins/sources/ostree.py b/buildstream/plugins/sources/ostree.py
index b311e24..f5f948b 100644
--- a/buildstream/plugins/sources/ostree.py
+++ b/buildstream/plugins/sources/ostree.py
@@ -122,6 +122,14 @@ class OSTreeSource(Source):
raise SourceError("{}: Failed to fetch ref '{}' from origin: {}\n\n{}"
.format(self, self.ref, self.url, e)) from e
+ def update_mirror(self):
+ self.ensure()
+ try:
+ _ostree.fetch(self.repo, progress=self.progress)
+ except OSTreeError as e:
+ raise SourceError("{}: Failed to fetch from origin: {}\n\n{}"
+ .format(self, self.url, e)) from e
+
def stage(self, directory):
self.ensure()
[buildstream] 09/10: Add mirror tasks to stream
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch valentindavid/update_mirror
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e0ba2b91cfd6d5378c4b05ec13b42106a7f7c767
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Apr 26 10:14:26 2018 +0200
Add mirror tasks to stream
---
buildstream/_context.py | 7 ++++-
buildstream/_frontend/app.py | 1 +
buildstream/_frontend/widget.py | 1 +
buildstream/_scheduler/__init__.py | 1 +
.../_scheduler/{__init__.py => mirrorqueue.py} | 30 +++++++++++++++-------
buildstream/_scheduler/queue.py | 3 +++
buildstream/_scheduler/scheduler.py | 3 ++-
buildstream/_stream.py | 26 ++++++++++++++++++-
buildstream/data/userconfig.yaml | 3 +++
9 files changed, 63 insertions(+), 12 deletions(-)
diff --git a/buildstream/_context.py b/buildstream/_context.py
index c0d49b2..e38cc76 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -95,6 +95,9 @@ class Context():
# Maximum number of push tasks
self.sched_pushers = 4
+ # Maximum number of mirror tasks
+ self.sched_mirrorers = 4
+
# Maximum number of retries for network tasks
self.sched_network_retries = 2
@@ -186,12 +189,14 @@ class Context():
scheduler = _yaml.node_get(defaults, Mapping, 'scheduler')
_yaml.node_validate(scheduler, [
'on-error', 'fetchers', 'builders',
- 'pushers', 'network-retries'
+ 'pushers', 'network-retries',
+ 'mirrorers'
])
self.sched_error_action = _yaml.node_get(scheduler, str, 'on-error')
self.sched_fetchers = _yaml.node_get(scheduler, int, 'fetchers')
self.sched_builders = _yaml.node_get(scheduler, int, 'builders')
self.sched_pushers = _yaml.node_get(scheduler, int, 'pushers')
+ self.sched_mirrorers = _yaml.node_get(scheduler, int, 'mirrorers')
self.sched_network_retries = _yaml.node_get(scheduler, int, 'network-retries')
# Load per-projects overrides
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index fa07a9a..d6af94a 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -193,6 +193,7 @@ class App():
'fetchers': 'sched_fetchers',
'builders': 'sched_builders',
'pushers': 'sched_pushers',
+ 'mirrorers': 'sched_mirrorers',
'network_retries': 'sched_network_retries'
}
for cli_option, context_attr in override_map.items():
diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py
index fe7229e..b0ea106 100644
--- a/buildstream/_frontend/widget.py
+++ b/buildstream/_frontend/widget.py
@@ -468,6 +468,7 @@ class LogLine(Widget):
values["Maximum Fetch Tasks"] = context.sched_fetchers
values["Maximum Build Tasks"] = context.sched_builders
values["Maximum Push Tasks"] = context.sched_pushers
+ values["Maximum Mirror Tasks"] = context.sched_mirrorers
values["Maximum Network Retries"] = context.sched_network_retries
text += self._format_values(values)
text += '\n'
diff --git a/buildstream/_scheduler/__init__.py b/buildstream/_scheduler/__init__.py
index 14cdebf..b5074b9 100644
--- a/buildstream/_scheduler/__init__.py
+++ b/buildstream/_scheduler/__init__.py
@@ -25,5 +25,6 @@ from .trackqueue import TrackQueue
from .buildqueue import BuildQueue
from .pushqueue import PushQueue
from .pullqueue import PullQueue
+from .mirrorqueue import MirrorQueue
from .scheduler import Scheduler, SchedStatus
diff --git a/buildstream/_scheduler/__init__.py b/buildstream/_scheduler/mirrorqueue.py
similarity index 54%
copy from buildstream/_scheduler/__init__.py
copy to buildstream/_scheduler/mirrorqueue.py
index 14cdebf..7d6226f 100644
--- a/buildstream/_scheduler/__init__.py
+++ b/buildstream/_scheduler/mirrorqueue.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
#
-# Copyright (C) 2017 Codethink Limited
+# Copyright (C) 2018 Codethink Limited
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -16,14 +16,26 @@
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
# Authors:
-# Tristan Van Berkom <tr...@codethink.co.uk>
+# Valentin David <va...@codethink.co.uk>
-from .queue import Queue, QueueStatus, QueueType
+from . import Queue, QueueType, QueueStatus
-from .fetchqueue import FetchQueue
-from .trackqueue import TrackQueue
-from .buildqueue import BuildQueue
-from .pushqueue import PushQueue
-from .pullqueue import PullQueue
-from .scheduler import Scheduler, SchedStatus
+class MirrorQueue(Queue):
+
+ action_name = "Mirror"
+ complete_name = "Mirrored"
+ queue_type = QueueType.MIRROR
+
+ def process(self, element):
+ for source in element.sources():
+ source.update_mirror()
+
+ def status(self, element):
+ if not list(element.sources()):
+ return QueueStatus.SKIP
+
+ return QueueStatus.READY
+
+ def done(self, element, result, success):
+ return success
diff --git a/buildstream/_scheduler/queue.py b/buildstream/_scheduler/queue.py
index 7c4ad69..4305cf4 100644
--- a/buildstream/_scheduler/queue.py
+++ b/buildstream/_scheduler/queue.py
@@ -45,6 +45,9 @@ class QueueType():
# Tasks which upload stuff to the internet
PUSH = 3
+ # Tasks which mirror sources
+ MIRROR = 4
+
# Queue status for a given element
#
diff --git a/buildstream/_scheduler/scheduler.py b/buildstream/_scheduler/scheduler.py
index 25e1e67..fa992b4 100644
--- a/buildstream/_scheduler/scheduler.py
+++ b/buildstream/_scheduler/scheduler.py
@@ -96,7 +96,8 @@ class Scheduler():
self._job_tokens = {
QueueType.FETCH: context.sched_fetchers,
QueueType.BUILD: context.sched_builders,
- QueueType.PUSH: context.sched_pushers
+ QueueType.PUSH: context.sched_pushers,
+ QueueType.MIRROR: context.sched_mirrorers
}
# run()
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index f2806b4..5ca146b 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -30,7 +30,7 @@ from tempfile import TemporaryDirectory
from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
from ._message import Message, MessageType
-from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
+from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue, MirrorQueue
from ._pipeline import Pipeline, PipelineSelection
from ._platform import Platform
from . import utils, _yaml, _site
@@ -213,6 +213,30 @@ class Stream():
self._enqueue_plan(elements)
self._run()
+ # mirror()
+ #
+ # Mirrors all sources on the pipeline.
+ #
+ # Args:
+ # targets (list of str): Targets to fetch
+ # selection (PipelineSelection): The selection mode for the specified targets
+ # except_targets (list of str): Specified targets to except from fetching
+ #
+ def mirror(self, targets, *,
+ selection=PipelineSelection.PLAN,
+ except_targets=None):
+ elements, _ = \
+ self._load(targets, (),
+ selection=selection,
+ except_targets=except_targets,
+ track_except_targets=except_targets,
+ fetch_subprojects=True)
+
+ mirrorqueue = MirrorQueue(self._scheduler)
+ self._add_queue(mirrorqueue)
+ self._enqueue_plan(elements, queue=mirrorqueue)
+ self._run()
+
# fetch()
#
# Fetches sources on the pipeline.
diff --git a/buildstream/data/userconfig.yaml b/buildstream/data/userconfig.yaml
index 6bb54ff..ce87ef8 100644
--- a/buildstream/data/userconfig.yaml
+++ b/buildstream/data/userconfig.yaml
@@ -36,6 +36,9 @@ scheduler:
# Maximum number of simultaneous artifact uploading tasks.
pushers: 4
+ # Maximum number of simultaneous mirror tasks.
+ mirrorers: 10
+
# Maximum number of retries for network tasks.
network-retries: 2
[buildstream] 01/10: buildstream/_loader/loadelement.py: Fix crash
with some elements that do not have depends
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch valentindavid/update_mirror
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f539ccbab5f3555fec6003bbdf985c18c0114231
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Apr 26 09:51:50 2018 +0200
buildstream/_loader/loadelement.py: Fix crash with some elements that do not have depends
---
buildstream/_loader/loadelement.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/buildstream/_loader/loadelement.py b/buildstream/_loader/loadelement.py
index b270fbe..247955e 100644
--- a/buildstream/_loader/loadelement.py
+++ b/buildstream/_loader/loadelement.py
@@ -167,6 +167,7 @@ def _extract_depends_from_node(node):
output_deps.append(dependency)
# Now delete "depends", we dont want it anymore
- del node[Symbol.DEPENDS]
+ if Symbol.DEPENDS in node:
+ del node[Symbol.DEPENDS]
return output_deps
[buildstream] 03/10:
buildstream/plugins/sources/_downloadablefilesource.py: Implement
update_mirror.
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch valentindavid/update_mirror
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 95fb47d9ae18552c7e4849118c22963ae29e85a3
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Apr 26 09:57:07 2018 +0200
buildstream/plugins/sources/_downloadablefilesource.py: Implement update_mirror.
Use namespaced mirrors in order to use expected filename for mirroring.
Add optional 'filename' configuration for mirroring in case it should
not be using the basename of the URL.
---
.../plugins/sources/_downloadablefilesource.py | 66 +++++++++++++++-------
1 file changed, 46 insertions(+), 20 deletions(-)
diff --git a/buildstream/plugins/sources/_downloadablefilesource.py b/buildstream/plugins/sources/_downloadablefilesource.py
index ec9c0fb..0e29c10 100644
--- a/buildstream/plugins/sources/_downloadablefilesource.py
+++ b/buildstream/plugins/sources/_downloadablefilesource.py
@@ -13,13 +13,18 @@ from buildstream import utils
class DownloadableFileSource(Source):
# pylint: disable=attribute-defined-outside-init
- COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
+ COMMON_CONFIG_KEYS = ['url', 'ref',
+ 'etag', 'filename']
+ COMMON_CONFIG_KEYS.extend(Source.COMMON_CONFIG_KEYS)
def configure(self, node):
self.original_url = self.node_get_member(node, str, 'url')
self.ref = self.node_get_member(node, str, 'ref', None)
self.url = self.translate_url(self.original_url)
self._warn_deprecated_etag(node)
+ self.filename = self.node_get_member(node, str, 'filename', None)
+ if self.filename is None:
+ self.filename = os.path.basename(self.url)
def preflight(self):
return
@@ -31,7 +36,7 @@ class DownloadableFileSource(Source):
if self.ref is None:
return Consistency.INCONSISTENT
- if os.path.isfile(self._get_mirror_file()):
+ if self._get_mirror_file():
return Consistency.CACHED
else:
@@ -70,7 +75,8 @@ class DownloadableFileSource(Source):
# file to be already cached because Source.fetch() will
# not be called if the source is already Consistency.CACHED.
#
- if os.path.isfile(self._get_mirror_file()):
+ cachedfile = self._get_mirror_file()
+ if cachedfile is not None and os.path.isfile(cachedfile):
return # pragma: nocover
# Download the file, raise hell if the sha256sums don't match,
@@ -87,21 +93,27 @@ class DownloadableFileSource(Source):
provenance = self.node_provenance(node, member_name='etag')
self.warn('{} "etag" is deprecated and ignored.'.format(provenance))
+ def update_mirror(self):
+ self._ensure_mirror()
+
def _get_etag(self, ref):
- etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref))
- if os.path.exists(etagfilename):
- with open(etagfilename, 'r') as etagfile:
- return etagfile.read()
+ mirrorfilename = self._get_mirror_file(sha=ref)
+ if mirrorfilename:
+ etagfilename = '{}.etag'.format(mirrorfilename)
+ if os.path.exists(etagfilename):
+ with open(etagfilename, 'r') as etagfile:
+ return etagfile.read()
return None
def _store_etag(self, ref, etag):
- etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref))
+ mirrorfilename = self._get_mirror_file(sha=ref, create=True)
+ etagfilename = '{}.etag'.format(mirrorfilename)
with utils.save_file_atomic(etagfilename) as etagfile:
etagfile.write(etag)
def _ensure_mirror(self):
- # Downloads from the url and caches it according to its sha256sum.
+ # Downloads from the url and caches it along with its sha256sum and etag.
try:
with self.tempdir() as td:
default_name = os.path.basename(self.url)
@@ -129,15 +141,15 @@ class DownloadableFileSource(Source):
with open(local_file, 'wb') as dest:
shutil.copyfileobj(response, dest)
- # Make sure url-specific mirror dir exists.
- if not os.path.isdir(self._get_mirror_dir()):
- os.makedirs(self._get_mirror_dir())
-
# Store by sha256sum
sha256 = utils.sha256sum(local_file)
# Even if the file already exists, move the new file over.
# In case the old file was corrupted somehow.
- os.rename(local_file, self._get_mirror_file(sha256))
+ mirrorfilename = self._get_mirror_file(sha=sha256, create=True)
+ os.rename(local_file, mirrorfilename)
+ if etag is not None:
+ with open('{}.etag'.format(mirrorfilename), 'w') as etagfile:
+ etagfile.write(etag)
if etag:
self._store_etag(sha256, etag)
@@ -156,9 +168,23 @@ class DownloadableFileSource(Source):
raise SourceError("{}: Error mirroring {}: {}"
.format(self, self.url, e)) from e
- def _get_mirror_dir(self):
- return os.path.join(self.get_mirror_directory(),
- utils.url_directory_name(self.original_url))
-
- def _get_mirror_file(self, sha=None):
- return os.path.join(self._get_mirror_dir(), sha or self.ref)
+ def _is_right_mirror(self, path, sha):
+ shafilename = os.path.join(path, '{}.sha256'.format(self.filename))
+ if os.path.exists(shafilename):
+ with open(shafilename, 'r') as shafile:
+ return shafile.read(64) == sha
+ else:
+ with open(shafilename, 'w') as shafile:
+ shafile.write(sha)
+ return True
+
+ def _get_mirror_file(self, sha=None, create=False):
+ if sha is None:
+ sha = self.ref
+ path = self.find_mirror_directory(self.original_url,
+ lambda path: self._is_right_mirror(path, sha),
+ create=create)
+ if path:
+ return os.path.join(path, self.filename)
+ else:
+ return None
[buildstream] 06/10: buildstream/plugins/sources/local.py:
Implement update_mirror (no-op).
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch valentindavid/update_mirror
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 45c1161f7cc0ed66488d36e6c1e24663d97ae775
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Apr 26 10:09:11 2018 +0200
buildstream/plugins/sources/local.py: Implement update_mirror (no-op).
---
buildstream/plugins/sources/local.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/buildstream/plugins/sources/local.py b/buildstream/plugins/sources/local.py
index 3193d10..6a96e9b 100644
--- a/buildstream/plugins/sources/local.py
+++ b/buildstream/plugins/sources/local.py
@@ -90,6 +90,10 @@ class LocalSource(Source):
# Nothing to do here for a local source
pass # pragma: nocover
+ def update_mirror(self):
+ # Nothing to do here for a local source
+ pass
+
def stage(self, directory):
# Dont use hardlinks to stage sources, they are not write protected
[buildstream] 02/10: buildstream/source.py: Add update_mirror()
method and utilities to have namespaced mirrors
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch valentindavid/update_mirror
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8a51f1ad439a97523893c7b053d46e4c96f7d311
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Apr 26 09:53:47 2018 +0200
buildstream/source.py: Add update_mirror() method and utilities to have namespaced mirrors
---
buildstream/source.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 73 insertions(+)
diff --git a/buildstream/source.py b/buildstream/source.py
index fa547d6..ee42d7a 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -247,6 +247,20 @@ class Source(Plugin):
"""
raise ImplError("Source plugin '{}' does not implement fetch()".format(self.get_kind()))
+ def update_mirror(self):
+ """Mirror remote sources locally. Its difference with :meth:`.fetch`
+ is that it will download everything from remote source,
+ whether or not the required `ref` depends on it.
+
+ Raises:
+ :class:`.SourceError`
+
+ Implementors should raise :class:`.SourceError` if the there is some
+ network error.
+ """
+
+ raise ImplError("Source plugin '{}' does not implement update_mirror()".format(self.get_kind()))
+
def stage(self, directory):
"""Stage the sources to a directory
@@ -287,6 +301,65 @@ class Source(Plugin):
#############################################################
# Public Methods #
#############################################################
+ def find_mirror_directory(self, upstream_url, predicate,
+ create=False):
+ """Find directory where source plugin data is stored.
+
+ Args:
+ upstream_url (str): url of upstream
+ predicate (function): predicate which takes a path and
+ should return if this path is
+ useable.
+ create (bool): whether no available space is found, one is
+ created.
+
+ Returns:
+ (str): The directory belonging to this source, or None if
+ not available.
+ """
+
+ base = self.get_mirror_directory()
+ subdir = utils.url_directory_name(upstream_url)
+ namespace = 0
+ while True:
+ curdir = os.path.join(base, subdir, str(namespace))
+ if not os.path.isdir(curdir):
+ if create:
+ os.makedirs(curdir, exist_ok=False)
+ if predicate(curdir):
+ return curdir
+ else:
+ return None
+ else:
+ return None
+ else:
+ if predicate(curdir):
+ return curdir
+ namespace += 1
+
+ def last_mirror_directory(self, upstream_url):
+ """Find last directory where source plugin has stored
+ data.
+
+ Args:
+ upstream_url (str): url of upstream
+
+ Returns:
+ (str): The directory belonging to this source, or None if
+ not available.
+ """
+
+ base = self.get_mirror_directory()
+ subdir = utils.url_directory_name(upstream_url)
+ namespace = 0
+ lastdir = None
+ while True:
+ curdir = os.path.join(base, subdir, str(namespace))
+ if not os.path.isdir(curdir):
+ return lastdir
+ lastdir = curdir
+ namespace += 1
+
def get_mirror_directory(self):
"""Fetches the directory where this source should store things
[buildstream] 04/10: buildstream/plugins/sources/git.py: Implement
update_mirror.
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch valentindavid/update_mirror
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ff6292eae24c531cc0d6b0b08dd308d50e778da5
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Apr 26 10:04:02 2018 +0200
buildstream/plugins/sources/git.py: Implement update_mirror.
Also disable gc in mirrors so that commits in removed or rewritten branches
are not deleted.
---
buildstream/plugins/sources/git.py | 131 +++++++++++++++++++++----------------
1 file changed, 76 insertions(+), 55 deletions(-)
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index f178656..fcf4f73 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -85,12 +85,11 @@ GIT_MODULES = '.gitmodules'
#
class GitMirror():
- def __init__(self, source, path, url, ref):
+ def __init__(self, source, path, url):
self.source = source
self.path = path
self.url = source.translate_url(url)
- self.ref = ref
self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(self.url))
# Ensures that the mirror exists
@@ -116,27 +115,34 @@ class GitMirror():
raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}'"
.format(self.source, self.url, tmpdir, self.mirror)) from e
- def fetch(self):
- self.source.call([self.source.host_git, 'fetch', 'origin', '--prune'],
+ self.source.call([self.source.host_git, 'config', 'gc.auto', '0'],
+ cwd=self.mirror,
+ fail="Failed to disable garbage collection on git repository {}".format(self.url))
+
+ def fetch_ref(self, ref):
+ ret = self.source.call([self.source.host_git, 'fetch', 'origin', ref],
+ cwd=self.mirror)
+ if ret != 0:
+ raise SourceError("Failed to fetch {} from remote git repository: {}".format(ref, self.url))
+
+ def update_mirror(self):
+ self.source.call([self.source.host_git, 'fetch', 'origin'],
fail="Failed to fetch from remote git repository: {}".format(self.url),
cwd=self.mirror)
- def has_ref(self):
- if not self.ref:
- return False
-
+ def has_ref(self, ref):
# If the mirror doesnt exist, we also dont have the ref
if not os.path.exists(self.mirror):
return False
# Check if the ref is really there
- rc = self.source.call([self.source.host_git, 'cat-file', '-t', self.ref], cwd=self.mirror)
+ rc = self.source.call([self.source.host_git, 'cat-file', '-t', ref], cwd=self.mirror)
return rc == 0
- def assert_ref(self):
- if not self.has_ref():
+ def assert_ref(self, ref):
+ if not self.has_ref(ref):
raise SourceError("{}: expected ref '{}' was not found in git repository: '{}'"
- .format(self.source, self.ref, self.url))
+ .format(self.source, ref, self.url))
def latest_commit(self, tracking):
_, output = self.source.check_output(
@@ -145,7 +151,7 @@ class GitMirror():
cwd=self.mirror)
return output.rstrip('\n')
- def stage(self, directory):
+ def stage(self, directory, ref):
fullpath = os.path.join(directory, self.path)
# We need to pass '--no-hardlinks' because there's nothing to
@@ -154,11 +160,11 @@ class GitMirror():
self.source.call([self.source.host_git, 'clone', '--no-checkout', '--no-hardlinks', self.mirror, fullpath],
fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath))
- self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
- fail="Failed to checkout git ref {}".format(self.ref),
+ self.source.call([self.source.host_git, 'checkout', '--force', ref],
+ fail="Failed to checkout git ref {}".format(ref),
cwd=fullpath)
- def init_workspace(self, directory):
+ def init_workspace(self, directory, ref):
fullpath = os.path.join(directory, self.path)
self.source.call([self.source.host_git, 'clone', '--no-checkout', self.mirror, fullpath],
@@ -168,13 +174,13 @@ class GitMirror():
fail='Failed to add remote origin "{}"'.format(self.url),
cwd=fullpath)
- self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
- fail="Failed to checkout git ref {}".format(self.ref),
+ self.source.call([self.source.host_git, 'checkout', '--force', ref],
+ fail="Failed to checkout git ref {}".format(ref),
cwd=fullpath)
# List the submodules (path/url tuples) present at the given ref of this repo
- def submodule_list(self):
- modules = "{}:{}".format(self.ref, GIT_MODULES)
+ def submodule_list(self, ref):
+ modules = "{}:{}".format(ref, GIT_MODULES)
exit_code, output = self.source.check_output(
[self.source.host_git, 'show', modules], cwd=self.mirror)
@@ -185,7 +191,7 @@ class GitMirror():
elif exit_code != 0:
raise SourceError(
"{plugin}: Failed to show gitmodules at ref {ref}".format(
- plugin=self, ref=self.ref))
+ plugin=self, ref=ref))
content = '\n'.join([l.strip() for l in output.splitlines()])
@@ -203,9 +209,7 @@ class GitMirror():
# Fetch the ref which this mirror requires its submodule to have,
# at the given ref of this mirror.
- def submodule_ref(self, submodule, ref=None):
- if not ref:
- ref = self.ref
+ def submodule_ref(self, submodule, ref):
# list objects in the parent repo tree to find the commit
# object that corresponds to the submodule
@@ -241,14 +245,14 @@ class GitSource(Source):
# pylint: disable=attribute-defined-outside-init
def configure(self, node):
- ref = self.node_get_member(node, str, 'ref', None)
+ self.ref = self.node_get_member(node, str, 'ref', None)
config_keys = ['url', 'track', 'ref', 'submodules', 'checkout-submodules']
self.node_validate(node, config_keys + Source.COMMON_CONFIG_KEYS)
self.original_url = self.node_get_member(node, str, 'url')
- self.mirror = GitMirror(self, '', self.original_url, ref)
self.tracking = self.node_get_member(node, str, 'track', None)
+ self.mirror = GitMirror(self, '', self.original_url)
self.checkout_submodules = self.node_get_member(node, bool, 'checkout-submodules', True)
self.submodules = []
@@ -273,7 +277,7 @@ class GitSource(Source):
# Here we want to encode the local name of the repository and
# the ref, if the user changes the alias to fetch the same sources
# from another location, it should not effect the cache key.
- key = [self.original_url, self.mirror.ref]
+ key = [self.original_url, self.ref]
# Only modify the cache key with checkout_submodules if it's something
# other than the default behaviour.
@@ -291,20 +295,20 @@ class GitSource(Source):
return key
def get_consistency(self):
- if self.have_all_refs():
+ if self.have_ref_and_submodules():
return Consistency.CACHED
- elif self.mirror.ref is not None:
+ elif self.ref is not None:
return Consistency.RESOLVED
return Consistency.INCONSISTENT
def load_ref(self, node):
- self.mirror.ref = self.node_get_member(node, str, 'ref', None)
+ self.ref = self.node_get_member(node, str, 'ref', None)
def get_ref(self):
- return self.mirror.ref
+ return self.ref
def set_ref(self, ref, node):
- node['ref'] = self.mirror.ref = ref
+ node['ref'] = self.ref = ref
def track(self):
@@ -316,12 +320,12 @@ class GitSource(Source):
.format(self.tracking, self.mirror.url),
silent_nested=True):
self.mirror.ensure()
- self.mirror.fetch()
+ self.mirror.fetch_ref(self.tracking)
# Update self.mirror.ref and node.ref from the self.tracking branch
ret = self.mirror.latest_commit(self.tracking)
- return ret
+ return ret
def fetch(self):
@@ -330,10 +334,10 @@ class GitSource(Source):
# Here we are only interested in ensuring that our mirror contains
# the self.mirror.ref commit.
self.mirror.ensure()
- if not self.mirror.has_ref():
- self.mirror.fetch()
+ if not self.mirror.has_ref(self.ref):
+ self.mirror.fetch_ref(self.ref)
- self.mirror.assert_ref()
+ self.mirror.assert_ref(self.ref)
# Here after performing any fetches, we need to also ensure that
# we've cached the desired refs in our mirrors of submodules.
@@ -341,14 +345,26 @@ class GitSource(Source):
self.refresh_submodules()
self.fetch_submodules()
+ def update_mirror(self):
+ if self.checkout_submodules:
+ self.warn("{}: Mirroring of submodules will not happen unless "
+ "submodules are explicitly listed."
+ .format(self.mirror.source))
+ for path, url in self.submodule_overrides.items():
+ if self.submodule_checkout_overrides.get(path, False):
+ mirror = GitMirror(self, path, url)
+ mirror.update_mirror()
+
+ self.mirror.update_mirror()
+
def init_workspace(self, directory):
# XXX: may wish to refactor this as some code dupe with stage()
self.refresh_submodules()
with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
- self.mirror.init_workspace(directory)
- for mirror in self.submodules:
- mirror.init_workspace(directory)
+ self.mirror.init_workspace(directory, self.ref)
+ for mirror, ref in self.submodules:
+ mirror.init_workspace(directory, ref)
def stage(self, directory):
@@ -362,28 +378,28 @@ class GitSource(Source):
# Stage the main repo in the specified directory
#
with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True):
- self.mirror.stage(directory)
- for mirror in self.submodules:
+ self.mirror.stage(directory, self.ref)
+ for mirror, ref in self.submodules:
if mirror.path in self.submodule_checkout_overrides:
checkout = self.submodule_checkout_overrides[mirror.path]
else:
checkout = self.checkout_submodules
if checkout:
- mirror.stage(directory)
+ mirror.stage(directory, ref)
###########################################################
# Local Functions #
###########################################################
- def have_all_refs(self):
- if not self.mirror.has_ref():
+ def have_ref_and_submodules(self):
+ if self.ref is None or not self.mirror.has_ref(self.ref):
return False
self.refresh_submodules()
- for mirror in self.submodules:
+ for mirror, ref in self.submodules:
if not os.path.exists(mirror.mirror):
return False
- if not mirror.has_ref():
+ if not mirror.has_ref(ref):
return False
return True
@@ -393,13 +409,18 @@ class GitSource(Source):
# Assumes that we have our mirror and we have the ref which we point to
#
def refresh_submodules(self):
+
+ if self.ref is None:
+ self.submodules = []
+ return
+
submodules = []
# XXX Here we should issue a warning if either:
# A.) A submodule exists but is not defined in the element configuration
# B.) The element configuration configures submodules which dont exist at the current ref
#
- for path, url in self.mirror.submodule_list():
+ for path, url in self.mirror.submodule_list(self.ref):
# Allow configuration to override the upstream
# location of the submodules.
@@ -407,10 +428,10 @@ class GitSource(Source):
if override_url:
url = override_url
- ref = self.mirror.submodule_ref(path)
+ ref = self.mirror.submodule_ref(path, self.ref)
if ref is not None:
- mirror = GitMirror(self, path, url, ref)
- submodules.append(mirror)
+ mirror = GitMirror(self, path, url)
+ submodules.append((mirror, ref))
self.submodules = submodules
@@ -421,11 +442,11 @@ class GitSource(Source):
# referred to at the given commit of the main git source.
#
def fetch_submodules(self):
- for mirror in self.submodules:
+ for mirror, ref in self.submodules:
mirror.ensure()
- if not mirror.has_ref():
- mirror.fetch()
- mirror.assert_ref()
+ if not mirror.has_ref(ref):
+ mirror.fetch_ref(ref)
+ mirror.assert_ref(ref)
# Plugin entry point
[buildstream] 07/10: buildstream/plugins/sources/patch.py:
Implement update_mirror (no-op).
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch valentindavid/update_mirror
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0e483cc02f4ebaef528443a184871e9623befde9
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Apr 26 10:09:42 2018 +0200
buildstream/plugins/sources/patch.py: Implement update_mirror (no-op).
---
buildstream/plugins/sources/patch.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/buildstream/plugins/sources/patch.py b/buildstream/plugins/sources/patch.py
index c9e40b1..2b5988f 100644
--- a/buildstream/plugins/sources/patch.py
+++ b/buildstream/plugins/sources/patch.py
@@ -82,6 +82,10 @@ class PatchSource(Source):
# Nothing to do here for a local source
pass # pragma: nocover
+ def update_mirror(self):
+ # Nothing to do here for a local source
+ pass
+
def stage(self, directory):
with self.timed_activity("Applying local patch: {}".format(self.path)):