You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:49:57 UTC
[buildstream] 01/01: Add `bst source push/pull` commands
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tlater/source-pushll
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9fbfa9b1e02ea64f9967aa1136fa75c2b3a30446
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Fri Aug 23 11:55:00 2019 +0100
Add `bst source push/pull` commands
---
src/buildstream/_frontend/cli.py | 84 ++++++++++++++++++++++
src/buildstream/_scheduler/__init__.py | 1 +
.../_scheduler/queues/sourcepullqueue.py | 43 +++++++++++
src/buildstream/_stream.py | 48 ++++++++++++-
src/buildstream/element.py | 16 +++++
tests/frontend/completions.py | 2 +
6 files changed, 193 insertions(+), 1 deletion(-)
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 1365ced..8f0e1fb 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -664,6 +664,90 @@ def source():
"""Manipulate sources for an element"""
+@source.command(name="push", short_help="Push an element's sources")
+@click.option('--deps', '-d', default='none',
+ type=click.Choice(['none', 'all']),
+ help='The dependencies to push (default: none)')
+@click.option('--remote', '-r', default=None,
+ help="The URL of the remote cache (defaults to the first configured cache)")
+@click.argument('elements', nargs=-1,
+ type=click.Path(readable=False))
+@click.pass_obj
+def source_push(app, elements, deps, remote):
+ """Push an element's sources to a remote source cache.
+
+ Specifying no elements will result in pushing the default targets
+ of the project. If no default targets are configured, all project
+ elements will be pushed.
+
+ When this command is executed from a workspace directory, the default
+ is to push the workspace element.
+
+ The default destination is the highest priority configured cache. You can
+ override this by passing a different cache URL with the `--remote` flag.
+
+ Specify `--deps` to control which sources to push:
+
+ \b
+ none: No dependencies, just the element itself
+ all: All dependencies
+ """
+ with app.initialized(session_name="Push"):
+ ignore_junction_targets = False
+
+ if not elements:
+ elements = app.project.get_default_targets()
+ # Junction elements cannot be pushed, exclude them from default targets
+ ignore_junction_targets = True
+
+ app.stream.push_sources(elements, selection=deps, remote=remote,
+ ignore_junction_targets=ignore_junction_targets)
+
+
+@source.command(name="pull", short_help="Pull an element's sources")
+@click.option('--deps', '-d', default='none',
+ type=click.Choice(['none', 'all']),
+ help='The dependency sources to pull (default: none)')
+@click.option('--remote', '-r', default=None,
+ help="The URL of the remote cache (defaults to the first configured cache)")
+@click.argument('elements', nargs=-1,
+ type=click.Path(readable=False))
+@click.pass_obj
+def source_pull(app, elements, deps, remote):
+ """Pull sources from the configured remote source cache.
+
+ Specifying no elements will result in pulling the default targets
+ of the project. If no default targets are configured, all project
+ elements will be pulled.
+
+ When this command is executed from a workspace directory, the default
+ is to pull the workspace element.
+
+ By default the sources will be pulled from one of the configured
+ caches if possible, following the usual priority order. If the
+ `--remote` flag is given, only the specified cache will be
+ queried.
+
+ Specify `--deps` to control which sources to pull:
+
+ \b
+ none: No dependencies, just the element itself
+ all: All dependencies
+
+ """
+
+ with app.initialized(session_name="Pull"):
+ ignore_junction_targets = False
+
+ if not elements:
+ elements = app.project.get_default_targets()
+ # Junction elements cannot be pulled, exclude them from default targets
+ ignore_junction_targets = True
+
+ app.stream.pull_sources(elements, selection=deps, remote=remote,
+ ignore_junction_targets=ignore_junction_targets)
+
+
##################################################################
# Source Fetch Command #
##################################################################
diff --git a/src/buildstream/_scheduler/__init__.py b/src/buildstream/_scheduler/__init__.py
index d2f458f..123f329 100644
--- a/src/buildstream/_scheduler/__init__.py
+++ b/src/buildstream/_scheduler/__init__.py
@@ -21,6 +21,7 @@ from .queues import Queue, QueueStatus
from .queues.fetchqueue import FetchQueue
from .queues.sourcepushqueue import SourcePushQueue
+from .queues.sourcepullqueue import SourcePullQueue
from .queues.trackqueue import TrackQueue
from .queues.buildqueue import BuildQueue
from .queues.artifactpushqueue import ArtifactPushQueue
diff --git a/src/buildstream/_scheduler/queues/sourcepullqueue.py b/src/buildstream/_scheduler/queues/sourcepullqueue.py
new file mode 100644
index 0000000..255edf4
--- /dev/null
+++ b/src/buildstream/_scheduler/queues/sourcepullqueue.py
@@ -0,0 +1,43 @@
+#
+# Copyright (C) 2019 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from . import Queue, QueueStatus
+from ..resources import ResourceType
+from ..._exceptions import SkipJob
+
+
+# A queue which pushes staged sources
+#
+class SourcePullQueue(Queue):
+
+ action_name = "Src-pull"
+ complete_name = "Sources pulled"
+ resources = [ResourceType.DOWNLOAD, ResourceType.CACHE]
+
+ def get_process_func(self):
+ return SourcePullQueue._pull_or_skip
+
+ def status(self, element):
+ if element._skip_source_pull():
+ return QueueStatus.SKIP
+
+ return QueueStatus.READY
+
+ @staticmethod
+ def _pull_or_skip(element):
+ if not element._source_pull():
+ raise SkipJob(SourcePullQueue.action_name)
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 453670a..9f37925 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -36,7 +36,7 @@ from ._artifactelement import verify_artifact_ref, ArtifactElement
from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError
from ._message import Message, MessageType
from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, \
- SourcePushQueue, BuildQueue, PullQueue, ArtifactPushQueue
+ SourcePushQueue, SourcePullQueue, BuildQueue, PullQueue, ArtifactPushQueue
from ._pipeline import Pipeline, PipelineSelection
from ._profile import Topics, PROFILER
from ._state import State
@@ -311,6 +311,52 @@ class Stream():
self._enqueue_plan(elements)
self._run()
+ def push_sources(self, targets, *,
+ selection=PipelineSelection.NONE,
+ ignore_junction_targets=False,
+ remote=None):
+ use_config = True
+ if remote:
+ use_config = False
+
+ elements, _ = self._load(targets, (),
+ selection=selection,
+ ignore_junction_targets=ignore_junction_targets,
+ source_remote_url=remote,
+ use_source_config=use_config)
+
+ if not self._sourcecache.has_push_remotes():
+ raise StreamError("No source caches available for pushing sources")
+
+ self._scheduler.clear_queues()
+ push_queue = SourcePushQueue(self._scheduler)
+ self._add_queue(push_queue)
+ self._enqueue_plan(elements, queue=push_queue)
+ self._run()
+
+ def pull_sources(self, targets, *,
+ selection=PipelineSelection.NONE,
+ ignore_junction_targets=False,
+ remote=None):
+ use_config = True
+ if remote:
+ use_config = False
+
+ elements, _ = self._load(targets, (),
+ selection=selection,
+ ignore_junction_targets=ignore_junction_targets,
+ source_remote_url=remote,
+ use_source_config=use_config)
+
+ if not self._sourcecache.has_fetch_remotes():
+ raise StreamError("No source caches available for pulling sources")
+
+ self._scheduler.clear_queues()
+ pull_queue = SourcePullQueue(self._scheduler)
+ self._add_queue(push_queue)
+ self._enqueue_plan(elements, queue=push_queue)
+ self._run()
+
# fetch()
#
# Fetches sources on the pipeline.
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index ca573be..0bac846 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1836,6 +1836,22 @@ class Element(Plugin):
# Notify successfull download
return True
+ def _skip_source_pull(self):
+ if not self.__sources or self._get_workspace():
+ return True
+ return self._source_cached() or not self.__sourcecache.has_fetch_remotes(plugin=self)
+
+ def _source_pull(self):
+ if self.__sourcecache.has_fetch_remotes(plugin=self) and not self._source_cached():
+ for source in self.sources():
+ if self.__sourcecache.pull(source):
+ # Once we find a cache with the source and manage
+ # to pull from it, we're done
+ return True
+
+ # If no caches gave us the source, this failed
+ return False
+
def _skip_source_push(self):
if not self.__sources or self._get_workspace():
return True
diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py
index e9fa25b..c0ea473 100644
--- a/tests/frontend/completions.py
+++ b/tests/frontend/completions.py
@@ -56,6 +56,8 @@ MAIN_OPTIONS = [
SOURCE_COMMANDS = [
'checkout ',
'fetch ',
+ 'pull ',
+ 'push ',
'track ',
]