You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 07:33:06 UTC
[buildstream] branch jennis/new_artifact_subcommands created (now
b8aea71)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch jennis/new_artifact_subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at b8aea71 cascache.py: Ensure path exists before trying to update the mtime
This branch includes the following new commits:
new ab6740f cli: Add artifact delete subcommand
new e992e41 artifactcache.py: Add prune() method to ArtifactCache
new 1050e97 cli.py: Defer pruning until all specified refs are removed
new cc2f295 tests: Add new tests for bst artifact delete
new 38a09bf artifactcache.py: API Cleanup - Add contains_ref() method
new 0ca62cf cli.py: Print to stderr when trying to delete an uncached artifact
new 0bdb5c1 cli.py: Ensure that both 'weak' and 'strong' artifacts are deleted
new b8aea71 cascache.py: Ensure path exists before trying to update the mtime
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 05/08: artifactcache.py: API Cleanup - Add
contains_ref() method
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/new_artifact_subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 38a09bf32ff4f80e8111ec88dfbec69b1cb1bc69
Author: James Ennis <ja...@codethink.com>
AuthorDate: Fri Jan 11 15:49:48 2019 +0000
artifactcache.py: API Cleanup - Add contains_ref() method
This ensures that we can determine whether an artifact is locally cached
(as well as an element) using the ArtifactCache API.
---
buildstream/_artifactcache.py | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 7bd5a53..24e6a3a 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -407,18 +407,30 @@ class ArtifactCache():
# contains():
#
- # Check whether the artifact for the specified Element is already available
- # in the local artifact cache.
+ # Check whether the (project state) artifact of the specified Element is
+ # already available in the local artifact cache.
#
# Args:
# element (Element): The Element to check
# key (str): The cache key to use
#
- # Returns: True if the artifact is in the cache, False otherwise
+ # Returns: True if the Element's (project state) artifact is in the cache,
+ # False otherwise
#
def contains(self, element, key):
ref = self.get_artifact_fullname(element, key)
+ return self.contains_ref(ref)
+ # contains_ref():
+ #
+ # Check whether an artifact is already available in the local artifact cache.
+ #
+ # Args:
+ # ref (str): The ref to check
+ #
+ # Returns: True if the artifact is in the cache, False otherwise
+ #
+ def contains_ref(self, ref):
return self.cas.contains(ref)
# contains_subdir_artifact():
[buildstream] 08/08: cascache.py: Ensure path exists before trying
to update the mtime
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/new_artifact_subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b8aea71b04b9a2c369997f606b5de25e7e00a85c
Author: James Ennis <ja...@codethink.com>
AuthorDate: Fri Jan 11 11:14:34 2019 +0000
cascache.py: Ensure path exists before trying to update the mtime
If an artifact is pulled from the cache without its buildtree,
CASCache.prune() will fail when it tries to update the mtimes
of the build tree's object files.
A new integration test has been added to tests/integration/artifact.py
which reflects this.
---
buildstream/_cas/cascache.py | 3 +++
tests/integration/artifact.py | 40 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 43 insertions(+)
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index f3eeb88..8aa2862 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -740,6 +740,9 @@ class CASCache():
if tree.hash in reachable:
return
+ if not os.path.exists(self.objpath(tree)):
+ return
+
if update_mtime:
os.utime(self.objpath(tree))
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index 607919e..59f6387 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -180,3 +180,43 @@ def test_artifact_delete_unbuilt_artifact(cli, tmpdir, datafiles):
expected_err = 'WARNING: {}, not found in local cache - no delete required\n'.format(artifact)
assert result.stderr == expected_err
+
+
+# Test that an artifact pulled from it's remote cache (without it's buildtree) will not
+# throw an Exception when trying to prune the cache.
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+def test_artifact_delete_pulled_artifact_without_buildtree(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ element = 'autotools/amhello.bst'
+
+ # Set up remote and local shares
+ local_cache = os.path.join(str(tmpdir), 'artifacts')
+ with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as remote:
+ cli.configure({
+ 'artifacts': {'url': remote.repo, 'push': True},
+ 'artifactdir': local_cache,
+ })
+
+ # Build the element
+ result = cli.run(project=project, args=['build', element])
+ result.assert_success()
+
+ # Make sure it's in the share
+ cache_key = cli.get_element_key(project, element)
+ assert remote.has_artifact('test', element, cache_key)
+
+ # Delete and then pull the artifact (without its buildtree)
+ result = cli.run(project=project, args=['artifact', 'delete', element])
+ result.assert_success()
+ assert cli.get_element_state(project, element) != 'cached'
+ result = cli.run(project=project, args=['pull', element])
+ result.assert_success()
+ assert cli.get_element_state(project, element) == 'cached'
+
+ # Now delete it again (it should have been pulled without the buildtree, but
+ # a digest of the buildtree is pointed to in the artifact's metadata
+ result = cli.run(project=project, args=['artifact', 'delete', element])
+ result.assert_success()
+ assert cli.get_element_state(project, element) != 'cached'
[buildstream] 04/08: tests: Add new tests for bst artifact delete
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/new_artifact_subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit cc2f2957e2c50ae8b07bf56ad882b238107d5c9d
Author: James Ennis <ja...@codethink.com>
AuthorDate: Fri Jan 11 15:47:34 2019 +0000
tests: Add new tests for bst artifact delete
These tests require a sandbox to build and have been marked to
skip accordingly.
---
tests/integration/artifact.py | 91 ++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 89 insertions(+), 2 deletions(-)
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index 2e12e71..ccd66db 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -21,8 +21,8 @@
import os
import pytest
-from tests.testutils import cli_integration as cli
-
+from tests.testutils import cli_integration as cli, create_artifact_share
+from tests.testutils.site import HAVE_BWRAP, IS_LINUX
pytestmark = pytest.mark.integration
@@ -66,3 +66,90 @@ def test_artifact_log(cli, tmpdir, datafiles):
assert result.exit_code == 0
# The artifact is cached under both a strong key and a weak key
assert (log + log) == result.output
+
+
+# Test that we can delete the artifact of the element which corresponds
+# to the current project state
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+def test_artifact_delete_element(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ element = 'integration.bst'
+
+ # Build the element and ensure it's cached
+ result = cli.run(project=project, args=['build', element])
+ result.assert_success()
+ assert cli.get_element_state(project, element) == 'cached'
+
+ result = cli.run(project=project, args=['artifact', 'delete', element])
+ result.assert_success()
+ assert cli.get_element_state(project, element) != 'cached'
+
+
+# Test that we can delete an artifact by specifying its ref.
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+def test_artifact_delete_artifact(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ element = 'integration.bst'
+
+ # Configure a local cache
+ local_cache = os.path.join(str(tmpdir), 'artifacts')
+ cli.configure({'artifactdir': local_cache})
+
+ # First build an element so that we can find its artifact
+ result = cli.run(project=project, args=['build', element])
+ result.assert_success()
+
+ # Obtain the artifact ref
+ cache_key = cli.get_element_key(project, element)
+ artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
+
+ # Explicitly check that the ARTIFACT exists in the cache
+ assert os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
+
+ # Delete the artifact
+ result = cli.run(project=project, args=['artifact', 'delete', artifact])
+ result.assert_success()
+
+ # Check that the ARTIFACT is no longer in the cache
+ assert not os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
+
+
+# Test the `bst artifact delete` command with multiple, different arguments.
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ element = 'integration.bst'
+ dep = 'base/base-alpine.bst'
+
+ # Configure a local cache
+ local_cache = os.path.join(str(tmpdir), 'artifacts')
+ cli.configure({'artifactdir': local_cache})
+
+ # First build an element so that we can find its artifact
+ result = cli.run(project=project, args=['build', element])
+ result.assert_success()
+ assert cli.get_element_state(project, element) == 'cached'
+ assert cli.get_element_state(project, dep) == 'cached'
+
+ # Obtain the artifact ref
+ cache_key = cli.get_element_key(project, element)
+ artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
+
+ # Explicitly check that the ARTIFACT exists in the cache
+ assert os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
+
+ # Delete the artifact
+ result = cli.run(project=project, args=['artifact', 'delete', artifact, dep])
+ result.assert_success()
+
+ # Check that the ARTIFACT is no longer in the cache
+ assert not os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
+
+ # Check that the dependency ELEMENT is no longer cached
+ assert cli.get_element_state(project, dep) != 'cached'
[buildstream] 01/08: cli: Add artifact delete subcommand
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/new_artifact_subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ab6740f9b34e773b512370fd05d9d112e4347d38
Author: Richard Maw <ri...@codethink.co.uk>
AuthorDate: Wed Dec 12 18:37:53 2018 +0000
cli: Add artifact delete subcommand
---
buildstream/_frontend/cli.py | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index b3e48d5..28e6027 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -1080,6 +1080,36 @@ def artifact_log(app, artifacts):
click.echo_via_pager(data)
+###################################################################
+# Artifact Delete Command #
+###################################################################
+@artifact.command(name='delete', short_help="Delete matching artifacts")
+@click.argument('artifacts', type=click.Path(), nargs=-1)
+@click.pass_obj
+def artifact_delete(app, artifacts):
+ '''Delete matching artifacts from the cache'''
+ from .._pipeline import PipelineSelection
+
+ with app.initialized():
+ cache = app.context.artifactcache
+
+ elements, artifacts = _classify_artifacts(artifacts, cache.cas,
+ app.project.directory)
+
+ if not elements and not artifacts:
+ element = app.context.guess_element()
+ if element is not None:
+ elements = [element]
+
+ if elements:
+ elements = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
+ for element in elements:
+ cache.remove(cache.get_artifact_fullname(element, element._get_cache_key()))
+ if artifacts:
+ for i, ref in enumerate(artifacts, start=1):
+ cache.cas.remove(ref, defer_prune=(i != len(artifacts)))
+
+
##################################################################
# DEPRECATED Commands #
##################################################################
[buildstream] 03/08: cli.py: Defer pruning until all specified refs
are removed
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/new_artifact_subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1050e97ca0b87ca2b7cd48ee4b9e1d135904cd54
Author: James Ennis <ja...@codethink.com>
AuthorDate: Fri Jan 11 12:13:15 2019 +0000
cli.py: Defer pruning until all specified refs are removed
---
buildstream/_frontend/cli.py | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 28e6027..f2bf45c 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -1101,13 +1101,19 @@ def artifact_delete(app, artifacts):
if element is not None:
elements = [element]
+ # Remove specified elements and artifacts
if elements:
elements = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
for element in elements:
- cache.remove(cache.get_artifact_fullname(element, element._get_cache_key()))
+ cache_key = element._get_cache_key()
+ ref = cache.get_artifact_fullname(element, cache_key)
+ cache.remove(ref, defer_prune=True)
if artifacts:
- for i, ref in enumerate(artifacts, start=1):
- cache.cas.remove(ref, defer_prune=(i != len(artifacts)))
+ for ref in artifacts:
+ cache.remove(ref, defer_prune=True)
+
+ # Now we've removed all the refs, prune the unreachable objects
+ cache.prune()
##################################################################
[buildstream] 02/08: artifactcache.py: Add prune() method to
ArtifactCache
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/new_artifact_subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e992e41cbda69af7453c2903e8c414b5f87951d4
Author: James Ennis <ja...@codethink.com>
AuthorDate: Fri Jan 11 11:47:47 2019 +0000
artifactcache.py: Add prune() method to ArtifactCache
Removal/pruning should be handled by ArtifactCache API. We should
not be using CASCache methods to remove BuildStream's artifacts.
---
buildstream/_artifactcache.py | 19 ++++++++++++++++---
buildstream/_cas/cascache.py | 1 -
2 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 16bde73..7bd5a53 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -462,8 +462,7 @@ class ArtifactCache():
# (int|None) The amount of space pruned from the repository in
# Bytes, or None if defer_prune is True
#
- def remove(self, ref):
-
+ def remove(self, ref, *, defer_prune=False):
# Remove extract if not used by other ref
tree = self.cas.resolve_ref(ref)
ref_name, ref_hash = os.path.split(ref)
@@ -482,7 +481,21 @@ class ArtifactCache():
if remove_extract:
utils._force_rmtree(extract)
- return self.cas.remove(ref)
+ return self.cas.remove(ref, defer_prune=defer_prune)
+
+ # prune():
+ #
+ # Prunes the artifact cache of objects which are unreachable from
+ # the repo
+ #
+ # Args:
+ # None
+ #
+ # Returns:
+ # (int) The amount of space pruned from the repository in bytes
+ #
+ def prune(self):
+ return self.cas.prune()
# extract():
#
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index adbd34c..f3eeb88 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -543,7 +543,6 @@ class CASCache():
# Bytes, or None if defer_prune is True
#
def remove(self, ref, *, defer_prune=False):
-
# Remove cache ref
refpath = self._refpath(ref)
if not os.path.exists(refpath):
[buildstream] 07/08: cli.py: Ensure that both 'weak' and 'strong'
artifacts are deleted
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/new_artifact_subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0bdb5c111340713e88c97dc5ba29bca63bb502a3
Author: James Ennis <ja...@codethink.com>
AuthorDate: Fri Jan 11 17:24:07 2019 +0000
cli.py: Ensure that both 'weak' and 'strong' artifacts are deleted
---
buildstream/_frontend/cli.py | 27 +++++++++++++++------------
1 file changed, 15 insertions(+), 12 deletions(-)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 828ea0a..6002547 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -7,6 +7,7 @@ from tempfile import TemporaryDirectory
import click
from .. import _yaml
+from ..types import _KeyStrength
from .._exceptions import BstError, LoadError, AppError
from .._versions import BST_FORMAT_VERSION
from .complete import main_bashcomplete, complete_path, CompleteUnhandled
@@ -1105,18 +1106,20 @@ def artifact_delete(app, artifacts):
if elements:
elements = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
for element in elements:
- cache_key = element._get_cache_key()
- ref = cache.get_artifact_fullname(element, cache_key)
- if cache.contains(element, cache_key):
- cache.remove(ref, defer_prune=True)
- click.echo("Removed {}.".format(ref))
- else:
- # If the ref is not present when we try to delete it, we should
- # not fail but just continue to delete. The pruning will take care
- # of any unreachable objects.
- click.echo("WARNING: {}, not found in local cache - no delete required"
- .format(ref), err=True)
- continue
+ cache_keys = set([element._get_cache_key(),
+ element._get_cache_key(strength=_KeyStrength.WEAK)])
+ for cache_key in cache_keys:
+ ref = cache.get_artifact_fullname(element, cache_key)
+ if cache.contains(element, cache_key):
+ cache.remove(ref, defer_prune=True)
+ click.echo("Removed {}.".format(ref))
+ else:
+ # If the ref is not present when we try to delete it, we should
+ # not fail but just continue to delete. The pruning will take care
+ # of any unreachable objects.
+ click.echo("WARNING: {}, not found in local cache - no delete required"
+ .format(ref), err=True)
+ continue
if artifacts:
for ref in artifacts:
[buildstream] 06/08: cli.py: Print to stderr when trying to delete
an uncached artifact
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/new_artifact_subcommands
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0ca62cf42f03df8f6c46beca8eeadd15cb2aac99
Author: James Ennis <ja...@codethink.com>
AuthorDate: Fri Jan 11 15:53:10 2019 +0000
cli.py: Print to stderr when trying to delete an uncached artifact
If we specify multiple elements/artifacts for deletion, we should not
throw an exception, and more importantly, stop, when we come across
an artifact that is uncached.
---
buildstream/_frontend/cli.py | 23 +++++++++++++++++++++--
tests/integration/artifact.py | 27 +++++++++++++++++++++++++++
2 files changed, 48 insertions(+), 2 deletions(-)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index f2bf45c..828ea0a 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -1107,10 +1107,29 @@ def artifact_delete(app, artifacts):
for element in elements:
cache_key = element._get_cache_key()
ref = cache.get_artifact_fullname(element, cache_key)
- cache.remove(ref, defer_prune=True)
+ if cache.contains(element, cache_key):
+ cache.remove(ref, defer_prune=True)
+ click.echo("Removed {}.".format(ref))
+ else:
+ # If the ref is not present when we try to delete it, we should
+ # not fail but just continue to delete. The pruning will take care
+ # of any unreachable objects.
+ click.echo("WARNING: {}, not found in local cache - no delete required"
+ .format(ref), err=True)
+ continue
+
if artifacts:
for ref in artifacts:
- cache.remove(ref, defer_prune=True)
+ if cache.contains_ref(ref):
+ cache.remove(ref, defer_prune=True)
+ click.echo("Removed {}.".format(ref))
+ else:
+ # If the ref is not present when we try to delete it, we should
+ # not fail but just continue to delete. The pruning will take care
+ # of any unreachable objects.
+ click.echo("WARNING: {}, not found in local cache - no delete required"
+ .format(ref), err=True)
+ continue
# Now we've removed all the refs, prune the unreachable objects
cache.prune()
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index ccd66db..607919e 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -153,3 +153,30 @@ def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles):
# Check that the dependency ELEMENT is no longer cached
assert cli.get_element_state(project, dep) != 'cached'
+
+
+# Test that we receive the appropriate stderr when we try to delete an artifact
+# that is not present in the cache.
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+def test_artifact_delete_unbuilt_artifact(cli, tmpdir, datafiles):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ element = 'integration.bst'
+
+ # Configure a local cache
+ local_cache = os.path.join(str(tmpdir), 'artifacts')
+ cli.configure({'artifactdir': local_cache})
+
+ # Ensure the element is not cached
+ assert cli.get_element_state(project, element) != 'cached'
+
+ # Obtain the artifact ref
+ cache_key = cli.get_element_key(project, element)
+ artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
+
+ # Try deleting the uncached artifact
+ result = cli.run(project=project, args=['artifact', 'delete', artifact])
+ result.assert_success()
+
+ expected_err = 'WARNING: {}, not found in local cache - no delete required\n'.format(artifact)
+ assert result.stderr == expected_err