You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:56:19 UTC
[buildstream] branch richardmaw/wip-artifact-subcommands created
(now 921f3d9)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch richardmaw/wip-artifact-subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 921f3d9 tests: Add tests for artifact group commands
This branch includes the following new commits:
new 5a68f31 cli: Add support for auto-completing artifact ref names
new b171213 cli: Add artifact command group
new ce9ec9a cli: Add artifact log command
new 921f3d9 tests: Add tests for artifact group commands
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 03/04: cli: Add artifact log command
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip-artifact-subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ce9ec9a595fa7ae01211d8c0e4dd8ce654462809
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Thu Nov 1 15:24:32 2018 +0000
cli: Add artifact log command
---
buildstream/_frontend/cli.py | 56 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 56 insertions(+)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index e07ec53..8dd1883 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -1,5 +1,7 @@
import os
import sys
+from contextlib import ExitStack
+from tempfile import TemporaryDirectory
import click
from .. import _yaml
@@ -889,3 +891,57 @@ def source_bundle(app, element, force, directory,
def artifact():
"""Manipulate cached Artifacts"""
pass
+
+
+################################################################
+# Artifact Log Command #
+################################################################
+@artifact.command(name='log', short_help="Show logs of an artifact")
+@click.option('-e', '--element', 'elements', help="Show logs for artifacts of this element",
+ type=click.Path(readable=False), multiple=True, required=False)
+@click.argument('artifacts', type=click.Path(), nargs=-1)
+@click.pass_obj
+def artifact_log(app, elements, artifacts):
+ """Show logs of all artifacts"""
+ from .._exceptions import CASError
+ from .._message import MessageType
+ from .._pipeline import PipelineSelection
+ from ..storage._casbaseddirectory import CasBasedDirectory
+
+ with ExitStack() as stack:
+ stack.enter_context(app.initialized())
+ cache = app.context.artifactcache
+
+ vdirs = []
+ extractdirs = []
+ if artifacts:
+ for ref in artifacts:
+ try:
+ vdir = CasBasedDirectory(cache.cas, cache.cas.resolve_ref(ref, update_mtime=True))
+ vdirs.append(vdir)
+ except CASError as e:
+ app._message(MessageType.WARN, "Artifact {} is not cached".format(ref), detail=str(e))
+ continue
+ if elements:
+ elements = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
+ for element in elements:
+ if not element._cached():
+ app._message(MessageType.WARN, "Element {} is not cached".format(element))
+ continue
+ ref = cache.get_artifact_fullname(element, element._get_cache_key())
+ vdir = CasBasedDirectory(cache.cas, cache.cas.resolve_ref(ref, update_mtime=True))
+ vdirs.append(vdir)
+
+ for vdir in vdirs:
+ # NOTE: If reading the logs feels unresponsive, here would be a good place to provide progress information.
+ logsdir = vdir.descend(["logs"])
+ td = stack.enter_context(TemporaryDirectory())
+ logsdir.export_files(td, can_link=True)
+ extractdirs.append(td)
+
+ for extractdir in extractdirs:
+ for log in (os.path.join(extractdir, log) for log in os.listdir(extractdir)):
+ # NOTE: Should click gain the ability to pass files to the pager this can be optimised.
+ with open(log) as f:
+ data = f.read()
+ click.echo_via_pager(data)
[buildstream] 02/04: cli: Add artifact command group
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip-artifact-subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b1712138dcb35981c0752ec088053cb097956233
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Thu Nov 1 15:23:20 2018 +0000
cli: Add artifact command group
---
buildstream/_frontend/cli.py | 9 +++++++++
tests/completions/completions.py | 1 +
2 files changed, 10 insertions(+)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index e3a88a5..e07ec53 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -880,3 +880,12 @@ def source_bundle(app, element, force, directory,
force=force,
compression=compression,
except_targets=except_)
+
+
+#############################################################
+# Artifact Command #
+#############################################################
+@cli.group(short_help="Manipulate cached Artifacts")
+def artifact():
+ """Manipulate cached Artifacts"""
+ pass
diff --git a/tests/completions/completions.py b/tests/completions/completions.py
index af35fb2..0966a52 100644
--- a/tests/completions/completions.py
+++ b/tests/completions/completions.py
@@ -6,6 +6,7 @@ from tests.testutils import cli
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
MAIN_COMMANDS = [
+ 'artifact ',
'build ',
'checkout ',
'fetch ',
[buildstream] 04/04: tests: Add tests for artifact group commands
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip-artifact-subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 921f3d9336a05f7cc01ce8092f60f7801a257f2a
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Thu Nov 1 15:25:48 2018 +0000
tests: Add tests for artifact group commands
---
tests/integration/artifact.py | 62 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 62 insertions(+)
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
new file mode 100644
index 0000000..f2a5cc6
--- /dev/null
+++ b/tests/integration/artifact.py
@@ -0,0 +1,62 @@
+#
+# Copyright (C) 2018 Codethink Limited
+# Copyright (C) 2018 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors: Richard Maw <ri...@codethink.co.uk>
+#
+
+import os
+import pytest
+
+from tests.testutils import cli_integration as cli
+
+
+pytestmark = pytest.mark.integration
+
+
+# Project directory
+DATA_DIR = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)),
+ "project",
+)
+
+
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+def test_artifact_log(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+
+ # Get the cache key of our test element
+ result = cli.run(project=project, silent=True, args=[
+ '--no-colors',
+ 'show', '--deps', 'none', '--format', '%{full-key}',
+ 'base.bst'
+ ])
+ key = result.output.strip()
+
+ # Ensure we have an artifact to read
+ result = cli.run(project=project, args=['build', 'base.bst'])
+ assert result.exit_code == 0
+
+ # Read the log via the element name
+ result = cli.run(project=project, args=['artifact', 'log', '-e', 'base.bst'])
+ assert result.exit_code == 0
+ log = result.output
+
+ # Read the log via the key
+ result = cli.run(project=project, args=['artifact', 'log', 'test/base/' + key])
+ assert result.exit_code == 0
+ assert log == result.output
[buildstream] 01/04: cli: Add support for auto-completing artifact
ref names
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch richardmaw/wip-artifact-subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5a68f313facba52d49639ea4ccce8275643788ca
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Thu Nov 1 15:22:48 2018 +0000
cli: Add support for auto-completing artifact ref names
---
buildstream/_frontend/cli.py | 28 +++++++++++++++++++++-------
1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index ef0dadb..e3a88a5 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -116,6 +116,18 @@ def complete_target(args, incomplete):
return complete_list
+def complete_artifact(args, incomplete):
+ from .._context import Context
+ ctx = Context()
+
+ config = None
+ for i, arg in enumerate(args):
+ if arg in ('-c', '--config'):
+ config = args[i + 1]
+ ctx.load(config)
+ return [ref for ref in ctx.artifactcache.cas.list_refs() if ref.startswith(incomplete)]
+
+
def override_completions(cmd, cmd_param, args, incomplete):
"""
:param cmd_param: command definition
@@ -130,13 +142,15 @@ def override_completions(cmd, cmd_param, args, incomplete):
# We can't easily extend click's data structures without
# modifying click itself, so just do some weak special casing
# right here and select which parameters we want to handle specially.
- if isinstance(cmd_param.type, click.Path) and \
- (cmd_param.name == 'elements' or
- cmd_param.name == 'element' or
- cmd_param.name == 'except_' or
- cmd_param.opts == ['--track'] or
- cmd_param.opts == ['--track-except']):
- return complete_target(args, incomplete)
+ if isinstance(cmd_param.type, click.Path):
+ if (cmd_param.name == 'elements' or
+ cmd_param.name == 'element' or
+ cmd_param.name == 'except_' or
+ cmd_param.opts == ['--track'] or
+ cmd_param.opts == ['--track-except']):
+ return complete_target(args, incomplete)
+ if cmd_param.name == 'artifacts':
+ return complete_artifact(args, incomplete)
raise CompleteUnhandled()