You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:57:31 UTC
[buildstream] 03/03: cli.py: Allow pull to handle artifact refs
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch jennis/deps_for_push
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8e654015c2eccd62d538ae6d39e78524913775b2
Author: James Ennis <ja...@codethink.co.uk>
AuthorDate: Thu Aug 29 12:32:21 2019 +0100
cli.py: Allow pull to handle artifact refs
This patch adds support for the handling of artifact refs in
bst artifact pull. A test for this has also been added.
---
src/buildstream/_frontend/cli.py | 10 +++++-----
src/buildstream/_project.py | 5 -----
src/buildstream/_stream.py | 6 ++++--
tests/frontend/buildcheckout.py | 2 +-
tests/frontend/pull.py | 42 ++++++++++++++++++++++++++++++++++++++++
5 files changed, 52 insertions(+), 13 deletions(-)
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 40d73e3..80b7479 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -1132,10 +1132,10 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression,
help='The dependency artifacts to pull (default: none)')
@click.option('--remote', '-r', default=None,
help="The URL of the remote cache (defaults to the first configured cache)")
-@click.argument('elements', nargs=-1,
+@click.argument('artifacts', nargs=-1,
type=click.Path(readable=False))
@click.pass_obj
-def artifact_pull(app, elements, deps, remote):
+def artifact_pull(app, artifacts, deps, remote):
"""Pull a built artifact from the configured remote artifact cache.
Specifying no elements will result in pulling the default targets
@@ -1159,12 +1159,12 @@ def artifact_pull(app, elements, deps, remote):
with app.initialized(session_name="Pull"):
ignore_junction_targets = False
- if not elements:
- elements = app.project.get_default_targets()
+ if not artifacts:
+ artifacts = app.project.get_default_targets()
# Junction elements cannot be pulled, exclude them from default targets
ignore_junction_targets = True
- app.stream.pull(elements, selection=deps, remote=remote,
+ app.stream.pull(artifacts, selection=deps, remote=remote,
ignore_junction_targets=ignore_junction_targets)
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 00beebf..7260d96 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -482,13 +482,8 @@ class Project():
# 2. The ArtifactCache.contains() method expects an Element
# and a key, not a ref.
#
- artifactdir = self._context.artifactdir
artifacts = []
for ref in targets:
- if not os.path.exists(os.path.join(artifactdir, ref)):
- raise LoadError("{}\nis not present in the artifact cache ({})".format(ref, artifactdir),
- LoadErrorReason.MISSING_FILE)
-
artifacts.append(ArtifactElement._new_from_artifact_ref(ref, self._context, task))
ArtifactElement._clear_artifact_refs_cache()
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index bea43f8..9abbb88 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -416,7 +416,8 @@ class Stream():
selection=selection,
ignore_junction_targets=ignore_junction_targets,
use_artifact_config=use_config,
- artifact_remote_url=remote)
+ artifact_remote_url=remote,
+ load_refs=True)
if not self._artifacts.has_fetch_remotes():
raise StreamError("No artifact caches available for pulling artifacts")
@@ -563,7 +564,8 @@ class Stream():
self._export_artifact(tar, location, compression, target, hardlinks, virdir)
except AttributeError as e:
raise ArtifactError("Artifact reference '{}' seems to be invalid. "
- "Note that an Element name can also be used.".format(artifact))from e
+ "Note that an Element name can also be used."
+ .format(artifact._element.get_artifact_name()))from e
else:
try:
with target._prepare_sandbox(scope=scope, directory=None,
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index 98b179b..6281217 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -358,7 +358,7 @@ def test_build_checkout_invalid_ref(datafiles, cli):
checkout_args = ['artifact', 'checkout', '--deps', 'none', '--tar', checkout, non_existent_artifact]
result = cli.run(project=project, args=checkout_args)
- assert "{}\nis not present in the artifact cache".format(non_existent_artifact) in result.stderr
+ assert "Artifact reference '{}' seems to be invalid".format(non_existent_artifact) in result.stderr
@pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index fd49ff1..f978258 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -558,3 +558,45 @@ def test_pull_access_rights(cli, tmpdir, datafiles):
st = os.lstat(os.path.join(checkout, 'usr/share/big-file'))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0644
+
+
+# Tests `bst artifact pull $artifact_ref`
+@pytest.mark.datafiles(DATA_DIR)
+def test_pull_artifact(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ element = 'target.bst'
+
+ # Configure a local cache
+ local_cache = os.path.join(str(tmpdir), 'cache')
+ cli.configure({'cachedir': local_cache})
+
+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+
+ # First build the target element and push to the remote.
+ cli.configure({
+ 'artifacts': {'url': share.repo, 'push': True}
+ })
+
+ result = cli.run(project=project, args=['build', element])
+ result.assert_success()
+
+ # Assert that the *artifact* is cached locally
+ cache_key = cli.get_element_key(project, element)
+ artifact_ref = os.path.join('test', os.path.splitext(element)[0], cache_key)
+ assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref))
+
+ # Assert that the target is shared (note that assert shared will use the artifact name)
+ assert_shared(cli, share, project, element)
+
+ # Now we've pushed, remove the local cache
+ shutil.rmtree(os.path.join(local_cache, 'artifacts'))
+
+ # Assert that nothing is cached locally anymore
+ assert not os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref))
+
+ # Now try bst artifact pull
+ result = cli.run(project=project, args=['artifact', 'pull', artifact_ref])
+ result.assert_success()
+
+ # And assert that it's again in the local cache, without having built
+ assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref))