You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:49:56 UTC
[buildstream] branch tlater/source-pushll created (now 9fbfa9b)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch tlater/source-pushll
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 9fbfa9b Add `bst source push/pull` commands
This branch includes the following new commits:
new 9fbfa9b Add `bst source push/pull` commands
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/01: Add `bst source push/pull` commands
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tlater/source-pushll
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9fbfa9b1e02ea64f9967aa1136fa75c2b3a30446
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Fri Aug 23 11:55:00 2019 +0100
Add `bst source push/pull` commands
---
src/buildstream/_frontend/cli.py | 84 ++++++++++++++++++++++
src/buildstream/_scheduler/__init__.py | 1 +
.../_scheduler/queues/sourcepullqueue.py | 43 +++++++++++
src/buildstream/_stream.py | 48 ++++++++++++-
src/buildstream/element.py | 16 +++++
tests/frontend/completions.py | 2 +
6 files changed, 193 insertions(+), 1 deletion(-)
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 1365ced..8f0e1fb 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -664,6 +664,90 @@ def source():
"""Manipulate sources for an element"""
+@source.command(name="push", short_help="Push an element's sources")
+@click.option('--deps', '-d', default='none',
+ type=click.Choice(['none', 'all']),
+ help='The dependencies to push (default: none)')
+@click.option('--remote', '-r', default=None,
+ help="The URL of the remote cache (defaults to the first configured cache)")
+@click.argument('elements', nargs=-1,
+ type=click.Path(readable=False))
+@click.pass_obj
+def source_push(app, elements, deps, remote):
+ """Push an element's sources to a remote source cache.
+
+ Specifying no elements will result in pushing the default targets
+ of the project. If no default targets are configured, all project
+ elements will be pushed.
+
+ When this command is executed from a workspace directory, the default
+ is to push the workspace element.
+
+ The default destination is the highest priority configured cache. You can
+ override this by passing a different cache URL with the `--remote` flag.
+
+ Specify `--deps` to control which sources to push:
+
+ \b
+ none: No dependencies, just the element itself
+ all: All dependencies
+ """
+ with app.initialized(session_name="Push"):
+ ignore_junction_targets = False
+
+ if not elements:
+ elements = app.project.get_default_targets()
+ # Junction elements cannot be pushed, exclude them from default targets
+ ignore_junction_targets = True
+
+ app.stream.push_sources(elements, selection=deps, remote=remote,
+ ignore_junction_targets=ignore_junction_targets)
+
+
+@source.command(name="pull", short_help="Pull an element's sources")
+@click.option('--deps', '-d', default='none',
+ type=click.Choice(['none', 'all']),
+ help='The dependency sources to pull (default: none)')
+@click.option('--remote', '-r', default=None,
+ help="The URL of the remote cache (defaults to the first configured cache)")
+@click.argument('elements', nargs=-1,
+ type=click.Path(readable=False))
+@click.pass_obj
+def source_pull(app, elements, deps, remote):
+ """Pull sources from the configured remote source cache.
+
+ Specifying no elements will result in pulling the default targets
+ of the project. If no default targets are configured, all project
+ elements will be pulled.
+
+ When this command is executed from a workspace directory, the default
+ is to pull the workspace element.
+
+ By default the sources will be pulled from one of the configured
+ caches if possible, following the usual priority order. If the
+ `--remote` flag is given, only the specified cache will be
+ queried.
+
+ Specify `--deps` to control which sources to pull:
+
+ \b
+ none: No dependencies, just the element itself
+ all: All dependencies
+
+ """
+
+ with app.initialized(session_name="Pull"):
+ ignore_junction_targets = False
+
+ if not elements:
+ elements = app.project.get_default_targets()
+ # Junction elements cannot be pulled, exclude them from default targets
+ ignore_junction_targets = True
+
+ app.stream.pull_sources(elements, selection=deps, remote=remote,
+ ignore_junction_targets=ignore_junction_targets)
+
+
##################################################################
# Source Fetch Command #
##################################################################
diff --git a/src/buildstream/_scheduler/__init__.py b/src/buildstream/_scheduler/__init__.py
index d2f458f..123f329 100644
--- a/src/buildstream/_scheduler/__init__.py
+++ b/src/buildstream/_scheduler/__init__.py
@@ -21,6 +21,7 @@ from .queues import Queue, QueueStatus
from .queues.fetchqueue import FetchQueue
from .queues.sourcepushqueue import SourcePushQueue
+from .queues.sourcepullqueue import SourcePullQueue
from .queues.trackqueue import TrackQueue
from .queues.buildqueue import BuildQueue
from .queues.artifactpushqueue import ArtifactPushQueue
diff --git a/src/buildstream/_scheduler/queues/sourcepullqueue.py b/src/buildstream/_scheduler/queues/sourcepullqueue.py
new file mode 100644
index 0000000..255edf4
--- /dev/null
+++ b/src/buildstream/_scheduler/queues/sourcepullqueue.py
@@ -0,0 +1,43 @@
+#
+# Copyright (C) 2019 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from . import Queue, QueueStatus
+from ..resources import ResourceType
+from ..._exceptions import SkipJob
+
+
+# A queue which pushes staged sources
+#
+class SourcePullQueue(Queue):
+
+ action_name = "Src-pull"
+ complete_name = "Sources pulled"
+ resources = [ResourceType.DOWNLOAD, ResourceType.CACHE]
+
+ def get_process_func(self):
+ return SourcePullQueue._pull_or_skip
+
+ def status(self, element):
+ if element._skip_source_pull():
+ return QueueStatus.SKIP
+
+ return QueueStatus.READY
+
+ @staticmethod
+ def _pull_or_skip(element):
+ if not element._source_pull():
+ raise SkipJob(SourcePullQueue.action_name)
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 453670a..9f37925 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -36,7 +36,7 @@ from ._artifactelement import verify_artifact_ref, ArtifactElement
from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError
from ._message import Message, MessageType
from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, \
- SourcePushQueue, BuildQueue, PullQueue, ArtifactPushQueue
+ SourcePushQueue, SourcePullQueue, BuildQueue, PullQueue, ArtifactPushQueue
from ._pipeline import Pipeline, PipelineSelection
from ._profile import Topics, PROFILER
from ._state import State
@@ -311,6 +311,52 @@ class Stream():
self._enqueue_plan(elements)
self._run()
+ def push_sources(self, targets, *,
+ selection=PipelineSelection.NONE,
+ ignore_junction_targets=False,
+ remote=None):
+ use_config = True
+ if remote:
+ use_config = False
+
+ elements, _ = self._load(targets, (),
+ selection=selection,
+ ignore_junction_targets=ignore_junction_targets,
+ source_remote_url=remote,
+ use_source_config=use_config)
+
+ if not self._sourcecache.has_push_remotes():
+ raise StreamError("No source caches available for pushing sources")
+
+ self._scheduler.clear_queues()
+ push_queue = SourcePushQueue(self._scheduler)
+ self._add_queue(push_queue)
+ self._enqueue_plan(elements, queue=push_queue)
+ self._run()
+
+ def pull_sources(self, targets, *,
+ selection=PipelineSelection.NONE,
+ ignore_junction_targets=False,
+ remote=None):
+ use_config = True
+ if remote:
+ use_config = False
+
+ elements, _ = self._load(targets, (),
+ selection=selection,
+ ignore_junction_targets=ignore_junction_targets,
+ source_remote_url=remote,
+ use_source_config=use_config)
+
+ if not self._sourcecache.has_fetch_remotes():
+ raise StreamError("No source caches available for pulling sources")
+
+ self._scheduler.clear_queues()
+ pull_queue = SourcePullQueue(self._scheduler)
+ self._add_queue(push_queue)
+ self._enqueue_plan(elements, queue=push_queue)
+ self._run()
+
# fetch()
#
# Fetches sources on the pipeline.
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index ca573be..0bac846 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1836,6 +1836,22 @@ class Element(Plugin):
# Notify successfull download
return True
+ def _skip_source_pull(self):
+ if not self.__sources or self._get_workspace():
+ return True
+ return self._source_cached() or not self.__sourcecache.has_fetch_remotes(plugin=self)
+
+ def _source_pull(self):
+ if self.__sourcecache.has_fetch_remotes(plugin=self) and not self._source_cached():
+ for source in self.sources():
+ if self.__sourcecache.pull(source):
+ # Once we find a cache with the source and manage
+ # to pull from it, we're done
+ return True
+
+ # If no caches gave us the source, this failed
+ return False
+
def _skip_source_push(self):
if not self.__sources or self._get_workspace():
return True
diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py
index e9fa25b..c0ea473 100644
--- a/tests/frontend/completions.py
+++ b/tests/frontend/completions.py
@@ -56,6 +56,8 @@ MAIN_OPTIONS = [
SOURCE_COMMANDS = [
'checkout ',
'fetch ',
+ 'pull ',
+ 'push ',
'track ',
]