You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ro...@apache.org on 2020/12/29 13:31:06 UTC
[buildstream] branch bst-pull created (now 3f0417c)
This is an automated email from the ASF dual-hosted git repository.
root pushed a change to branch bst-pull
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 3f0417c Add initial `bst pull` command
This branch includes the following new commits:
new e28852e element.py: Allow calling _skip_push() for uncached elements
new b86c9a5 Add initial `bst push` command
new 3f0417c Add initial `bst pull` command
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 03/03: Add initial `bst pull` command
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch bst-pull
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3f0417c9ffbe2e6f86ae180169908e92f9622133
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Tue Aug 1 14:22:02 2017 +0100
Add initial `bst pull` command
---
buildstream/_frontend/main.py | 31 +++++++++++++++++++++++++++++++
buildstream/_pipeline.py | 38 ++++++++++++++++++++++++++++++++++++++
2 files changed, 69 insertions(+)
diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index 43cdc43..4eb7851 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -209,6 +209,37 @@ def track(app, target, variant, deps, except_):
##################################################################
+# Pull Command #
+##################################################################
+@cli.command(short_help="Pull a built artifact")
+@click.option('--variant',
+ help='A variant of the specified target')
+@click.option('--deps', '-d', default='none',
+ type=click.Choice(['none', 'all']),
+ help='The dependencies to fetch (default: none)')
+@click.argument('target')
+@click.pass_obj
+def pull(app, target, variant, deps):
+ """Pull a built artifact from the configured remote artifact cache.
+
+ Specify `--deps` to control which artifacts to pull:
+
+ \b
+ none: No dependencies, just the element itself
+ all: All dependencies
+ """
+ app.initialize(target, variant)
+ try:
+ to_pull = app.pipeline.deps_elements(deps)
+ app.pipeline.pull(app.scheduler, to_pull)
+ click.echo("")
+ except _BstError as e:
+ click.echo("")
+ click.echo("ERROR: {}".format(e))
+ sys.exit(-1)
+
+
+##################################################################
# Push Command #
##################################################################
@cli.command(short_help="Push a built artifact")
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index 8092340..3c65e09 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -631,6 +631,44 @@ class Pipeline():
self.open_workspace(scheduler, workspace_dir, source_index, no_checkout,
track, False)
+ # pull()
+ #
+ # Pulls elements from the pipeline
+ #
+ # Args:
+ # scheduler (Scheduler): The scheduler to run this pipeline on
+ # elements (list): List of elements to pull
+ #
+ def pull(self, scheduler, elements):
+
+ if not self.artifacts.can_fetch():
+ self.message(self.target, MessageType.FAIL, "Not configured for pulling artifacts")
+
+ plan = elements
+ self.assert_consistent(plan)
+ self.session_elements = len(plan)
+
+ pull = PullQueue()
+ pull.enqueue(plan)
+ queues = [pull]
+
+ self.message(self.target, MessageType.START, "Pulling {} artifacts".format(len(plan)))
+ elapsed, status = scheduler.run(queues)
+ pulled = len(pull.processed_elements)
+
+ if status == SchedStatus.ERROR:
+ self.message(self.target, MessageType.FAIL, "Pull failed", elapsed=elapsed)
+ raise PipelineError()
+ elif status == SchedStatus.TERMINATED:
+ self.message(self.target, MessageType.WARN,
+ "Terminated after pulling {} elements".format(pulled),
+ elapsed=elapsed)
+ raise PipelineError()
+ else:
+ self.message(self.target, MessageType.SUCCESS,
+ "Pulled {} complete".format(pulled),
+ elapsed=elapsed)
+
# push()
#
# Pushes elements in the pipeline
[buildstream] 01/03: element.py: Allow calling _skip_push() for
uncached elements
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch bst-pull
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e28852ee2cf0aaafd4e17df9156e8e167e6c8d38
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Wed Aug 2 11:21:55 2017 +0100
element.py: Allow calling _skip_push() for uncached elements
This is required when using a push queue without build queue.
---
buildstream/element.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/buildstream/element.py b/buildstream/element.py
index 84cd9a5..09ead82 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1149,7 +1149,8 @@ class Element(Plugin):
# (bool): True if this element should not be pushed
#
def _skip_push(self):
- self._assert_cached()
+ if not self._cached(recalculate=None):
+ return True
# Do not push tained artifact
if self._tainted():
[buildstream] 02/03: Add initial `bst push` command
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch bst-pull
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b86c9a5cbff7335aa45e0a012e82026c41fde033
Author: Sam Thursfield <sa...@codethink.co.uk>
AuthorDate: Thu Jul 13 16:43:46 2017 +0000
Add initial `bst push` command
This is mainly useful for testing artifact caches and such. Most users
will hopefully be able to make use of artifact caches populated by
automated build machines, but right now it's unlikely that most people
will be pushing artifacts around.
---
buildstream/_frontend/main.py | 31 +++++++++++++++++++++++++++++++
buildstream/_pipeline.py | 38 ++++++++++++++++++++++++++++++++++++++
2 files changed, 69 insertions(+)
diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index 79adf11..43cdc43 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -209,6 +209,37 @@ def track(app, target, variant, deps, except_):
##################################################################
+# Push Command #
+##################################################################
+@cli.command(short_help="Push a built artifact")
+@click.option('--variant',
+ help='A variant of the specified target')
+@click.option('--deps', '-d', default='none',
+ type=click.Choice(['none', 'all']),
+ help='The dependencies to fetch (default: none)')
+@click.argument('target')
+@click.pass_obj
+def push(app, target, variant, deps):
+ """Push a built artifact to the configured remote artifact cache.
+
+ Specify `--deps` to control which artifacts to push:
+
+ \b
+ none: No dependencies, just the element itself
+ all: All dependencies
+ """
+ app.initialize(target, variant)
+ try:
+ to_push = app.pipeline.deps_elements(deps)
+ app.pipeline.push(app.scheduler, to_push)
+ click.echo("")
+ except _BstError as e:
+ click.echo("")
+ click.echo("ERROR: {}".format(e))
+ sys.exit(-1)
+
+
+##################################################################
# Show Command #
##################################################################
@cli.command(short_help="Show elements in the pipeline")
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index 64ce4d6..8092340 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -631,6 +631,44 @@ class Pipeline():
self.open_workspace(scheduler, workspace_dir, source_index, no_checkout,
track, False)
+ # push()
+ #
+ # Pushes elements in the pipeline
+ #
+ # Args:
+ # scheduler (Scheduler): The scheduler to run this pipeline on
+ # elements (list): List of elements to push
+ #
+ def push(self, scheduler, elements):
+
+ if not self.artifacts.can_push():
+ self.message(self.target, MessageType.FAIL, "Not configured for pushing artifacts")
+
+ plan = elements
+ self.assert_consistent(plan)
+ self.session_elements = len(plan)
+
+ push = PushQueue()
+ push.enqueue(plan)
+ queues = [push]
+
+ self.message(self.target, MessageType.START, "Pushing {} artifacts".format(len(plan)))
+ elapsed, status = scheduler.run(queues)
+ pushed = len(push.processed_elements)
+
+ if status == SchedStatus.ERROR:
+ self.message(self.target, MessageType.FAIL, "Push failed", elapsed=elapsed)
+ raise PipelineError()
+ elif status == SchedStatus.TERMINATED:
+ self.message(self.target, MessageType.WARN,
+ "Terminated after pushing {} elements".format(pushed),
+ elapsed=elapsed)
+ raise PipelineError()
+ else:
+ self.message(self.target, MessageType.SUCCESS,
+ "Pushed {} complete".format(pushed),
+ elapsed=elapsed)
+
# remove_elements():
#
# Internal function