You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 07:11:58 UTC
[buildstream] branch jennis/136-clean-remote-cache created (now
f6cab64)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at f6cab64 _ostree.py: Remove unused __list_remote_refs function
This branch includes the following new commits:
new 49d40f8 _ostree.py: Reintroduce remove()
new 8eb7d6a utils.py: Add get_dir_size helper
new bbc38f9 utils.py: Optimise get_scandir_dir_size()
new b031c1e utils.py: Optimise get_walk_dir_size()
new 82c7848 _ostree.py: Introduce list_all_refs() and list_artifacts()
new ac22a29 pushreceive.py: Ensure there is a repopath
new f990f88 Allow the user to set a cache quota
new 5c38fa0 Allow cache quota to be set with human friendly input e.g. 2K
new 361efba pushreceive.py: Remove LRP artifacts from cache, introduce clean_up_cache()
new 4d3676c element_generators.py: Add a utility function to create arbitrary size elements
new 7c194fe push.py: Add tests to test remote cache expiry
new c33fcc2 pushreceive.py: Add comments to code and name variables more sensibly
new 59d709d pushreceive.py: Ensure too large artifacts are not pushed
new 05e2395 _ostree.py: Adhere to private symbol policy
new f6cab64 _ostree.py: Remove unused __list_remote_refs function
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 08/15: Allow cache quota to be set with human
friendly input e.g. 2K
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5c38fa0ae26283333a9cd23a5a6192f6ccdd8265
Author: James Ennis <ja...@codethink.com>
AuthorDate: Thu Apr 26 16:07:37 2018 +0100
Allow cache quota to be set with human friendly input e.g. 2K
utils.py: Added _parse_size() function
pushreceive.py: Altered receive_main() so that it now accepts
human friendly input.
---
buildstream/_artifactcache/pushreceive.py | 14 ++++++----
buildstream/utils.py | 44 +++++++++++++++++++++++++++++++
doc/source/artifacts.rst | 6 +++--
3 files changed, 57 insertions(+), 7 deletions(-)
diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index d276c71..bcd2e4a 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -33,6 +33,7 @@ from urllib.parse import urlparse
import click
import gi
+from ..utils import _parse_size
from .. import _signals # nopep8
from .._profile import Topics, profile_start, profile_end
@@ -573,7 +574,11 @@ class OSTreeReceiver(object):
def __init__(self, repopath, pull_url, cache_quota):
self.repopath = repopath
self.pull_url = pull_url
- self.cache_quota = cache_quota
+ if cache_quota:
+ # Parse the string and find the quota in bytes (int)
+ self.cache_quota = _parse_size(cache_quota, repopath)
+ else:
+ self.cache_quota = None
if self.repopath is None:
self.repo = OSTree.Repo.new_default()
@@ -800,8 +805,9 @@ def push(repo, remote, branches, output):
@click.option('--debug', '-d', is_flag=True, default=False, help="Debug mode")
@click.option('--pull-url', type=str, required=True,
help="Clients who try to pull over SSH will be redirected here")
-@click.option('--cache-quota', type=int,
- help="Implement a quota on the cache (in bytes) here")
+@click.option('--cache-quota', type=str,
+ help="Implement a cache quota. Accepted suffixes are: K, M, G, T "
+ "or no suffix (for bytes).\nExample: --cache-quota 20G for 20 Gb")
@click.argument('repo')
def receive_main(verbose, debug, pull_url, cache_quota, repo):
"""A BuildStream sister program for receiving artifacts send to a shared artifact cache
@@ -814,7 +820,5 @@ def receive_main(verbose, debug, pull_url, cache_quota, repo):
logging.basicConfig(format='%(module)s: %(levelname)s: %(message)s',
level=loglevel, stream=sys.stderr)
- # IDEA: have it so that cache_quota can allow human friendly input.
-
receiver = OSTreeReceiver(repo, pull_url, cache_quota)
return receiver.run()
diff --git a/buildstream/utils.py b/buildstream/utils.py
index e6e5989..c1aee35 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -1126,3 +1126,47 @@ def _deduplicate(iterable, key=None):
if k not in seen:
seen_add(k)
yield element
+
+
+# _parse_size():
+#
+# Convert a string representing data size to a number of
+# bytes. E.g. "2K" -> 2048.
+#
+# This uses the same format as systemd's
+# [resource-control](https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html#).
+#
+# Arguments:
+# size (str) The string to parse
+# volume (str) A path on the volume to consider for percentage
+# specifications
+#
+# Returns:
+# (int|None) The number of bytes, or None if 'infinity' was specified.
+#
+# Raises:
+# UtilError if the string is not a valid data size.
+#
+def _parse_size(size, volume):
+ if size == 'infinity':
+ return None
+
+ matches = re.fullmatch(r'([0-9]\.?[0-9]*)([KMGT%]?)', size)
+ if matches is None:
+ raise UtilError("{} is not a valid data size."
+ "\nOptions are: K, M, G, T and %.".format(size))
+
+ num, unit = matches.groups()
+
+ if unit == '%':
+ num = float(num)
+ if num > 100:
+ raise UtilError("{}% is not a valid percentage value.".format(num))
+
+ stat_ = os.statvfs(volume)
+ disk_size = stat_.f_blocks * stat_.f_bsize
+
+ return disk_size * (num / 100)
+
+ units = ('', 'K', 'M', 'G', 'T')
+ return int(num) * 1024**units.index(unit)
diff --git a/doc/source/artifacts.rst b/doc/source/artifacts.rst
index f181c67..23a389f 100644
--- a/doc/source/artifacts.rst
+++ b/doc/source/artifacts.rst
@@ -160,8 +160,10 @@ For this you will want something like the following in your ``/etc/ssh/sshd_conf
# command must be specified here; 'artifacts' is
# the HOME relative path to the artifact cache.
# The exact pull URL must also be specified.
- # A cache-quota is optional and should be specified in bytes
- ## FIXME: The above should have a function which converts 20G -> bytes...
+ # A cache-quota is optional and can either be specified
+ # by the exact integer, or in human friendly form,
+ # for example, a 20 GB quota is specified with "20G"
+
ForceCommand bst-artifact-receive --pull-url https://example.com/artifacts --cache-quota quota_in_bytes --verbose artifacts
[buildstream] 15/15: _ostree.py: Remove unused __list_remote_refs
function
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f6cab6410e52f6e18a9e72bc4cf16c7d9185ac2b
Author: James Ennis <ja...@codethink.com>
AuthorDate: Tue May 22 10:41:15 2018 +0100
_ostree.py: Remove unused __list_remote_refs function
---
buildstream/_ostree.py | 20 --------------------
1 file changed, 20 deletions(-)
diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py
index 93fad13..72685a8 100644
--- a/buildstream/_ostree.py
+++ b/buildstream/_ostree.py
@@ -596,23 +596,3 @@ def __list_all_refs(repo):
return refs
except GLib.GError as e:
raise OSTreeError(message=e.message) from e
-
-
-# __list_remote_refs():
-#
-# Fetch list of refs from a remote.
-#
-# Args:
-# repo (OSTree.Repo): The repo
-# remote (str): An optional remote name, defaults to 'origin'
-#
-# Returns:
-# (dict): A dict of refs to checksums.
-#
-def __list_remote_refs(repo, remote="origin"):
- try:
- _, refs = repo.remote_list_refs(remote)
- return refs
- except GLib.GError as e:
- (_, remote_url) = repo.remote_get_url(remote)
- raise OSTreeError(message="{} when attempting to fetch from {}".format(e.message, remote_url)) from e
[buildstream] 03/15: utils.py: Optimise get_scandir_dir_size()
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit bbc38f92fab223a7442829871351c554d835f36e
Author: James Ennis <ja...@codethink.com>
AuthorDate: Tue May 22 16:35:50 2018 +0100
utils.py: Optimise get_scandir_dir_size()
---
buildstream/utils.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 60761dc..f22b57f 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -541,7 +541,7 @@ def save_file_atomic(filename, mode='w', *, buffering=-1, encoding=None,
# (str) The path whose size to check.
#
# Returns:
-# (int) The size on disk.
+# (int) The size on disk in bytes.
#
def _get_dir_size(path):
scandir = None
@@ -564,11 +564,10 @@ def _get_dir_size(path):
total = 0
for f in scandir(path):
+ total += f.stat(follow_symlinks=False).st_size
+
if f.is_dir(follow_symlinks=False):
- total += f.stat(follow_symlinks=False).st_size
total += get_scandir_dir_size(f.path)
- elif f.is_file(follow_symlinks=False):
- total += f.stat(follow_symlinks=False).st_size
return total
[buildstream] 04/15: utils.py: Optimise get_walk_dir_size()
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b031c1eaf2acadaa326eb26a30aba2286b3a293e
Author: James Ennis <ja...@codethink.com>
AuthorDate: Tue May 22 16:36:54 2018 +0100
utils.py: Optimise get_walk_dir_size()
---
buildstream/utils.py | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/buildstream/utils.py b/buildstream/utils.py
index f22b57f..e6e5989 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -578,13 +578,17 @@ def _get_dir_size(path):
for dirpath, dirs, files in os.walk(path):
for d in dirs:
path = os.path.join(dirpath, d)
- if os.path.exists(path):
- total += os.stat(path, follow_symlinks=False).st_size
+ try:
+ total += os.lstat(path).st_size
+ except FileNotFoundError:
+ pass
for f in files:
path = os.path.join(dirpath, f)
- if os.path.exists(path):
- total += os.stat(path, follow_symlinks=False).st_size
+ try:
+ total += os.lstat(path).st_size
+ except FileNotFoundError:
+ pass
return total
[buildstream] 07/15: Allow the user to set a cache quota
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f990f8888e06fea64ae5fea9f491046550a00d3c
Author: James Ennis <ja...@codethink.com>
AuthorDate: Wed Apr 18 15:51:32 2018 +0100
Allow the user to set a cache quota
---
buildstream/_artifactcache/pushreceive.py | 11 ++++++++---
doc/source/artifacts.rst | 4 +++-
2 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index 75a65e1..d276c71 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -570,9 +570,10 @@ class OSTreePusher(object):
# pull_url (str): Redirection for clients who want to pull, not push.
#
class OSTreeReceiver(object):
- def __init__(self, repopath, pull_url):
+ def __init__(self, repopath, pull_url, cache_quota):
self.repopath = repopath
self.pull_url = pull_url
+ self.cache_quota = cache_quota
if self.repopath is None:
self.repo = OSTree.Repo.new_default()
@@ -799,8 +800,10 @@ def push(repo, remote, branches, output):
@click.option('--debug', '-d', is_flag=True, default=False, help="Debug mode")
@click.option('--pull-url', type=str, required=True,
help="Clients who try to pull over SSH will be redirected here")
+@click.option('--cache-quota', type=int,
+ help="Implement a quota on the cache (in bytes) here")
@click.argument('repo')
-def receive_main(verbose, debug, pull_url, repo):
+def receive_main(verbose, debug, pull_url, cache_quota, repo):
"""A BuildStream sister program for receiving artifacts send to a shared artifact cache
"""
loglevel = logging.WARNING
@@ -811,5 +814,7 @@ def receive_main(verbose, debug, pull_url, repo):
logging.basicConfig(format='%(module)s: %(levelname)s: %(message)s',
level=loglevel, stream=sys.stderr)
- receiver = OSTreeReceiver(repo, pull_url)
+ # IDEA: have it so that cache_quota can allow human friendly input.
+
+ receiver = OSTreeReceiver(repo, pull_url, cache_quota)
return receiver.run()
diff --git a/doc/source/artifacts.rst b/doc/source/artifacts.rst
index e0808e8..f181c67 100644
--- a/doc/source/artifacts.rst
+++ b/doc/source/artifacts.rst
@@ -160,7 +160,9 @@ For this you will want something like the following in your ``/etc/ssh/sshd_conf
# command must be specified here; 'artifacts' is
# the HOME relative path to the artifact cache.
# The exact pull URL must also be specified.
- ForceCommand bst-artifact-receive --pull-url https://example.com/artifacts --verbose artifacts
+ # A cache-quota is optional and should be specified in bytes
+ ## FIXME: The above should have a function which converts 20G -> bytes...
+ ForceCommand bst-artifact-receive --pull-url https://example.com/artifacts --cache-quota quota_in_bytes --verbose artifacts
User configuration
[buildstream] 09/15: pushreceive.py: Remove LRP artifacts from
cache, introduce clean_up_cache()
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 361efbac89926950ebc9ae5f59944b876ca4b868
Author: James Ennis <ja...@codethink.com>
AuthorDate: Wed Apr 18 15:01:37 2018 +0100
pushreceive.py: Remove LRP artifacts from cache, introduce clean_up_cache()
---
buildstream/_artifactcache/pushreceive.py | 57 +++++++++++++++++++++++++++++--
1 file changed, 54 insertions(+), 3 deletions(-)
diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index bcd2e4a..c60011d 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -33,7 +33,8 @@ from urllib.parse import urlparse
import click
import gi
-from ..utils import _parse_size
+from ..utils import _get_dir_size, _parse_size
+from .. import _ostree
from .. import _signals # nopep8
from .._profile import Topics, profile_start, profile_end
@@ -278,7 +279,7 @@ class PushMessageReader(object):
_, args = self.receive([PushCommandType.update])
return args
- def receive_putobjects(self, repo):
+ def receive_putobjects(self, repo, repo_size, cache_quota):
received_objects = []
@@ -289,6 +290,7 @@ class PushMessageReader(object):
#
# This should block while tar.next() reads the next
# tar object from the stream.
+ current_repo_size = repo_size
while True:
filepos = tar.fileobj.tell()
tar_info = tar.next()
@@ -301,6 +303,11 @@ class PushMessageReader(object):
tar.fileobj.read(512)
break
+ # obtain size of tar object in bytes
+ artifact_size = tar_info.size
+ if cache_quota and artifact_size + current_repo_size > cache_quota:
+ current_repo_size = clean_up_cache(repo, current_repo_size, artifact_size, cache_quota)
+
tar.extract(tar_info, self.tmpdir)
received_objects.append(tar_info.name)
@@ -650,8 +657,16 @@ class OSTreeReceiver(object):
logging.debug('Received done before any objects, exiting')
return 0
+ # Determine the size (in bytes) of the artifact cache
+ # Expensive function, we should probably only work this out if there is a quota,
+ # otherwise we don't need it.
+ if self.cache_quota:
+ repo_size = _get_dir_size(self.repopath)
+ else:
+ repo_size = None
+
# Receive the actual objects
- received_objects = self.reader.receive_putobjects(self.repo)
+ received_objects = self.reader.receive_putobjects(self.repo, repo_size, self.cache_quota)
# Ensure that pusher has sent all objects
self.reader.receive_done()
@@ -800,6 +815,42 @@ def push(repo, remote, branches, output):
return False
+# clean_up_cache()
+#
+# Keep removing Least Recently Pushed (LRP) artifacts in a cache until there
+# is enough space for the incoming artifact
+#
+# Args:
+# repo: OSTree.Repo object
+# repo_size: The size of the repo in bytes
+# artifact_size: The size of the artifact in bytes
+# cache_quota: quota of the cache in bytes
+#
+# Returns:
+# int: The new repo_size
+#
+def clean_up_cache(repo, repo_size, artifact_size, cache_quota):
+ # obtain a list of LRP artifacts
+ LRP_artifacts = _ostree.list_artifacts(repo)
+
+ removed_size = 0 # in bytes
+ while artifact_size + repo_size - removed_size > cache_quota:
+ try:
+ to_remove = LRP_artifacts.pop(0) # The first element in the list is the LRP artifact
+ except IndexError:
+ logging.info("There are no more artifacts left in the cache. Adding artifact...")
+ break
+
+ removed_size += _ostree.remove(repo, to_remove, defer_prune=False)
+
+ if removed_size > 0:
+ logging.info("Successfully removed {} bytes from the cache".format(removed_size))
+ else:
+ logging.info("No artifacts were removed from the cache.")
+
+ return artifact_size + repo_size - removed_size
+
+
@click.command(short_help="Receive pushed artifacts over ssh")
@click.option('--verbose', '-v', is_flag=True, default=False, help="Verbose mode")
@click.option('--debug', '-d', is_flag=True, default=False, help="Debug mode")
[buildstream] 10/15: element_generators.py: Add a utility function
to create arbitrary size elements
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4d3676c3062125a459e2d36efe10fb3c98bbac7a
Author: James Ennis <ja...@codethink.com>
AuthorDate: Tue Apr 24 16:38:02 2018 +0100
element_generators.py: Add a utility function to create arbitrary size elements
New file consisting of functions that allow us to generate elements
on the fly
__init__.py now includes create_element_size()
---
tests/testutils/__init__.py | 1 +
tests/testutils/element_generators.py | 40 +++++++++++++++++++++++++++++++++++
2 files changed, 41 insertions(+)
diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py
index 7e5b792..93143b5 100644
--- a/tests/testutils/__init__.py
+++ b/tests/testutils/__init__.py
@@ -1,3 +1,4 @@
from .runcli import cli, cli_integration
from .repo import create_repo, ALL_REPO_KINDS
from .artifactshare import create_artifact_share
+from .element_generators import create_element_size
diff --git a/tests/testutils/element_generators.py b/tests/testutils/element_generators.py
new file mode 100644
index 0000000..3f6090d
--- /dev/null
+++ b/tests/testutils/element_generators.py
@@ -0,0 +1,40 @@
+import os
+
+from buildstream import _yaml
+
+
+# create_element_size()
+#
+# This will open a "<name>_data" file for writing and write
+# <size> MB of urandom (/dev/urandom) "stuff" into the file.
+# A bst import element file is then created: <name>.bst
+#
+# Args:
+# name: (str) of the element name (e.g. target.bst)
+# path: (str) pathway to the project/elements directory
+# dependencies: A list of strings (can also be an empty list)
+# size: (int) size of the element in bytes
+#
+# Returns:
+# Nothing (creates a .bst file of specified size)
+#
+def create_element_size(name, path, dependencies, size):
+ os.makedirs(path, exist_ok=True)
+
+ # Create a file to be included in this element's artifact
+ with open(os.path.join(path, name + '_data'), 'wb+') as f:
+ f.write(os.urandom(size))
+
+ # Simplest case: We want this file (of specified size) to just
+ # be an import element.
+ element = {
+ 'kind': 'import',
+ 'sources': [
+ {
+ 'kind': 'local',
+ 'path': os.path.join(path, name + '_data')
+ }
+ ],
+ 'depends': dependencies
+ }
+ _yaml.dump(element, os.path.join(path, name))
[buildstream] 13/15: pushreceive.py: Ensure too large artifacts are
not pushed
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 59d709d2196f2cdf8df847ebc51133b5134970bf
Author: James Ennis <ja...@codethink.com>
AuthorDate: Wed Apr 25 17:04:42 2018 +0100
pushreceive.py: Ensure too large artifacts are not pushed
---
buildstream/_artifactcache/pushreceive.py | 40 ++++++++++++++++++++++++-------
1 file changed, 31 insertions(+), 9 deletions(-)
diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index a8ba742..e13fd75 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -36,6 +36,7 @@ import gi
from ..utils import _get_dir_size, _parse_size
from .. import _ostree
from .. import _signals # nopep8
+from .._exceptions import ArtifactError
from .._profile import Topics, profile_start, profile_end
gi.require_version('OSTree', '1.0')
@@ -57,6 +58,11 @@ class PushExistsException(Exception):
pass
+# Trying to push an artifact that is too big for the cache
+class ArtifactTooBigError(Exception):
+ pass
+
+
class PushCommandType(Enum):
info = 0
update = 1
@@ -286,7 +292,6 @@ class PushMessageReader(object):
return args
def receive_putobjects(self, repo, repo_size, cache_quota):
-
received_objects = []
# Open a TarFile for reading uncompressed tar from a stream
@@ -311,11 +316,18 @@ class PushMessageReader(object):
# obtain size of tar object in bytes
artifact_size = tar_info.size
- if cache_quota and artifact_size + current_repo_size > cache_quota:
+ if cache_quota and artifact_size < cache_quota and artifact_size + current_repo_size > cache_quota:
current_repo_size = clean_up_cache(repo, current_repo_size, artifact_size, cache_quota)
- tar.extract(tar_info, self.tmpdir)
- received_objects.append(tar_info.name)
+ if cache_quota and artifact_size > cache_quota:
+ # Tell the user we've done nothing with this artifact
+ # This could potentially be from accidentially setting a small cache quota...
+ raise ArtifactTooBigError("Artifact of size: {} bytes is too big for the remote cache"
+ " under the current quota of: {} bytes."
+ .format(artifact_size, cache_quota))
+ else:
+ tar.extract(tar_info, self.tmpdir)
+ received_objects.append(tar_info.name)
# Finished with this stream
tar.close()
@@ -632,8 +644,10 @@ class OSTreeReceiver(object):
def run(self):
try:
exit_code = self.do_run()
- self.close()
- return exit_code
+ except ArtifactTooBigError:
+ logging.warning("The artifact was too big for the cache under the current quota,"
+ " so it has not been received")
+ exit_code = 0
except:
# BLIND EXCEPT - Just abort if we receive any exception, this
# can be a broken pipe, a tarfile read error when the remote
@@ -641,6 +655,9 @@ class OSTreeReceiver(object):
self.close()
raise
+ self.close()
+ return exit_code
+
def do_run(self):
# Receive remote info
args = self.reader.receive_info()
@@ -822,7 +839,13 @@ def push(repo, remote, branches, output):
return True
except ConnectionError as e:
# Connection attempt failed or connection was terminated unexpectedly
- terminate_push()
+ try:
+ terminate_push()
+ except BrokenPipeError:
+ # If the artifact is too big for the cache, we force the tar stream to quit
+ # So let's just log this and return push as False
+ logging.info("Ref {} is too big for the cache under the current quota".format(branches))
+ return False
raise PushException("Connection failed") from e
except PushException:
terminate_push()
@@ -858,8 +881,7 @@ def clean_up_cache(repo, repo_size, artifact_size, cache_quota):
try:
to_remove = LRP_artifacts.pop(0) # The first element in the list is the LRP artifact
except IndexError:
- logging.info("There are no more artifacts left in the cache. Adding artifact...")
- break
+ raise ArtifactError("There are no more artifacts in the cache.")
removed_size += _ostree.remove(repo, to_remove, defer_prune=False)
[buildstream] 14/15: _ostree.py: Adhere to private symbol policy
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 05e239572782aa9286e5c41df0adc38867849973
Author: James Ennis <ja...@codethink.com>
AuthorDate: Tue May 22 10:19:43 2018 +0100
_ostree.py: Adhere to private symbol policy
---
buildstream/_ostree.py | 60 +++++++++++++++++++++++++-------------------------
1 file changed, 30 insertions(+), 30 deletions(-)
diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py
index 080447b..93fad13 100644
--- a/buildstream/_ostree.py
+++ b/buildstream/_ostree.py
@@ -554,7 +554,33 @@ def configure_remote(repo, remote, url, key_url=None):
raise OSTreeError("Failed to add gpg key from url '{}': {}".format(key_url, e.message)) from e
-# list_all_refs():
+# list_artifacts():
+#
+# List cached artifacts in Least Recently Modified (LRM) order.
+#
+# Returns:
+# (list) - A list of refs in LRM order
+#
+def list_artifacts(repo):
+ # string of: /path/to/repo/refs/heads
+ ref_heads = os.path.join(repo.get_path().get_path(), 'refs', 'heads')
+
+ # obtain list of <project>/<element>/<key>
+ refs = __list_all_refs(repo).keys()
+
+ mtimes = []
+ for ref in refs:
+ ref_path = os.path.join(ref_heads, ref)
+ if os.path.exists(ref_path):
+ # Obtain the mtime (the time a file was last modified)
+ mtimes.append(os.path.getmtime(ref_path))
+
+ # NOTE: Sorted will sort from earliest to latest, thus the
+ # first element of this list will be the file modified earliest.
+ return [ref for _, ref in sorted(zip(mtimes, refs))]
+
+
+# __list_all_refs():
#
# Create a list of all refs.
#
@@ -564,7 +590,7 @@ def configure_remote(repo, remote, url, key_url=None):
# Returns:
# (dict): A dict of refs to checksums.
#
-def list_all_refs(repo):
+def __list_all_refs(repo):
try:
_, refs = repo.list_refs(None)
return refs
@@ -572,7 +598,7 @@ def list_all_refs(repo):
raise OSTreeError(message=e.message) from e
-# list_remote_refs():
+# __list_remote_refs():
#
# Fetch list of refs from a remote.
#
@@ -583,36 +609,10 @@ def list_all_refs(repo):
# Returns:
# (dict): A dict of refs to checksums.
#
-def list_remote_refs(repo, remote="origin"):
+def __list_remote_refs(repo, remote="origin"):
try:
_, refs = repo.remote_list_refs(remote)
return refs
except GLib.GError as e:
(_, remote_url) = repo.remote_get_url(remote)
raise OSTreeError(message="{} when attempting to fetch from {}".format(e.message, remote_url)) from e
-
-
-# list_artifacts():
-#
-# List cached artifacts in Least Recently Modified (LRM) order.
-#
-# Returns:
-# (list) - A list of refs in LRM order
-#
-def list_artifacts(repo):
- # string of: /path/to/repo/refs/heads
- ref_heads = os.path.join(repo.get_path().get_path(), 'refs', 'heads')
-
- # obtain list of <project>/<element>/<key>
- refs = list_all_refs(repo).keys()
-
- mtimes = []
- for ref in refs:
- ref_path = os.path.join(ref_heads, ref)
- if os.path.exists(ref_path):
- # Obtain the mtime (the time a file was last modified)
- mtimes.append(os.path.getmtime(ref_path))
-
- # NOTE: Sorted will sort from earliest to latest, thus the
- # first element of this list will be the file modified earliest.
- return [ref for _, ref in sorted(zip(mtimes, refs))]
[buildstream] 05/15: _ostree.py: Introduce list_all_refs() and
list_artifacts()
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 82c7848a41218e51765d7cf0aa379d5ae3b2194f
Author: Tristan Maat <tm...@tlater.net>
AuthorDate: Sat Mar 24 17:56:56 2018 +0000
_ostree.py: Introduce list_all_refs() and list_artifacts()
---
buildstream/_ostree.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 64 insertions(+)
diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py
index 246b54f..080447b 100644
--- a/buildstream/_ostree.py
+++ b/buildstream/_ostree.py
@@ -552,3 +552,67 @@ def configure_remote(repo, remote, url, key_url=None):
repo.remote_gpg_import(remote, stream, None, 0, None)
except GLib.GError as e:
raise OSTreeError("Failed to add gpg key from url '{}': {}".format(key_url, e.message)) from e
+
+
+# list_all_refs():
+#
+# Create a list of all refs.
+#
+# Args:
+# repo (OSTree.Repo): The repo
+#
+# Returns:
+# (dict): A dict of refs to checksums.
+#
+def list_all_refs(repo):
+ try:
+ _, refs = repo.list_refs(None)
+ return refs
+ except GLib.GError as e:
+ raise OSTreeError(message=e.message) from e
+
+
+# list_remote_refs():
+#
+# Fetch list of refs from a remote.
+#
+# Args:
+# repo (OSTree.Repo): The repo
+# remote (str): An optional remote name, defaults to 'origin'
+#
+# Returns:
+# (dict): A dict of refs to checksums.
+#
+def list_remote_refs(repo, remote="origin"):
+ try:
+ _, refs = repo.remote_list_refs(remote)
+ return refs
+ except GLib.GError as e:
+ (_, remote_url) = repo.remote_get_url(remote)
+ raise OSTreeError(message="{} when attempting to fetch from {}".format(e.message, remote_url)) from e
+
+
+# list_artifacts():
+#
+# List cached artifacts in Least Recently Modified (LRM) order.
+#
+# Returns:
+# (list) - A list of refs in LRM order
+#
+def list_artifacts(repo):
+ # string of: /path/to/repo/refs/heads
+ ref_heads = os.path.join(repo.get_path().get_path(), 'refs', 'heads')
+
+ # obtain list of <project>/<element>/<key>
+ refs = list_all_refs(repo).keys()
+
+ mtimes = []
+ for ref in refs:
+ ref_path = os.path.join(ref_heads, ref)
+ if os.path.exists(ref_path):
+ # Obtain the mtime (the time a file was last modified)
+ mtimes.append(os.path.getmtime(ref_path))
+
+ # NOTE: Sorted will sort from earliest to latest, thus the
+ # first element of this list will be the file modified earliest.
+ return [ref for _, ref in sorted(zip(mtimes, refs))]
[buildstream] 01/15: _ostree.py: Reintroduce remove()
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 49d40f896536c2263925e8c52b3a1db2b3e56509
Author: Tristan Maat <tm...@tlater.net>
AuthorDate: Fri Mar 23 17:26:32 2018 +0000
_ostree.py: Reintroduce remove()
---
buildstream/_ostree.py | 36 ++++++++++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py
index dfa7567..246b54f 100644
--- a/buildstream/_ostree.py
+++ b/buildstream/_ostree.py
@@ -225,6 +225,42 @@ def exists(repo, ref):
return has_object
+# remove():
+#
+# Removes the given commit or symbolic ref from the repo.
+#
+# Args:
+# repo (OSTree.Repo): The repo
+# ref (str): A commit checksum or symbolic ref
+# defer_prune (bool): Whether to defer pruning to the caller. NOTE:
+# The space won't be freed until you manually
+# call repo.prune.
+#
+# Returns:
+# (int|None) The amount of space pruned from the repository in
+# Bytes, or None if defer_prune is True
+#
+def remove(repo, ref, *, defer_prune=False):
+
+ # Get the commit checksum, this will:
+ #
+ # o Return a commit checksum if ref is a symbolic branch
+ # o Return the same commit checksum if ref is a valid commit checksum
+ # o Return None if the ostree repo doesnt know this ref.
+ #
+ check = checksum(repo, ref)
+ if check is None:
+ raise OSTreeError("Could not find artifact for ref '{}'".format(ref))
+
+ repo.set_ref_immediate(None, ref, None)
+
+ if not defer_prune:
+ _, _, _, pruned = repo.prune(OSTree.RepoPruneFlags.REFS_ONLY, -1)
+ return pruned
+
+ return None
+
+
# checksum():
#
# Returns the commit checksum for a given symbolic ref,
[buildstream] 02/15: utils.py: Add get_dir_size helper
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8eb7d6aa121c5009208daa1675963b66d42206d0
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Mar 20 13:19:13 2018 +0000
utils.py: Add get_dir_size helper
---
buildstream/utils.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 64 insertions(+)
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 8e72196..60761dc 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -533,6 +533,70 @@ def save_file_atomic(filename, mode='w', *, buffering=-1, encoding=None,
raise
+# _get_dir_size():
+#
+# Get the disk usage of a given directory in bytes.
+#
+# Arguments:
+# (str) The path whose size to check.
+#
+# Returns:
+# (int) The size on disk.
+#
+def _get_dir_size(path):
+ scandir = None
+ artifact_size = 0
+
+ # python3.4 does not have scandir, but it is significantly
+ # faster for this type of work.
+ if hasattr(os, 'scandir'):
+ from os import scandir # pylint: disable=no-name-in-module
+ else:
+ # If our user has scandir for python<3.5 installed, we use
+ # that, because why not?
+ try:
+ from scandir import scandir
+ except ImportError:
+ pass
+
+ def get_scandir_dir_size(path):
+ path = os.path.abspath(path)
+ total = 0
+
+ for f in scandir(path):
+ if f.is_dir(follow_symlinks=False):
+ total += f.stat(follow_symlinks=False).st_size
+ total += get_scandir_dir_size(f.path)
+ elif f.is_file(follow_symlinks=False):
+ total += f.stat(follow_symlinks=False).st_size
+
+ return total
+
+ def get_walk_dir_size(path):
+ path = os.path.abspath(path)
+ total = 0
+
+ for dirpath, dirs, files in os.walk(path):
+ for d in dirs:
+ path = os.path.join(dirpath, d)
+ if os.path.exists(path):
+ total += os.stat(path, follow_symlinks=False).st_size
+
+ for f in files:
+ path = os.path.join(dirpath, f)
+ if os.path.exists(path):
+ total += os.stat(path, follow_symlinks=False).st_size
+
+ return total
+
+ if scandir:
+ artifact_size = get_scandir_dir_size(path)
+ else:
+ artifact_size = get_walk_dir_size(path)
+
+ return artifact_size
+
+
# A sentinel to be used as a default argument for functions that need
# to distinguish between a kwarg set to None and an unset kwarg.
_sentinel = object()
[buildstream] 06/15: pushreceive.py: Ensure there is a repopath
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ac22a29e609da6262453f455777ea422769a647f
Author: James Ennis <ja...@codethink.com>
AuthorDate: Wed Apr 18 12:42:52 2018 +0100
pushreceive.py: Ensure there is a repopath
---
buildstream/_artifactcache/pushreceive.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index 777065e..75a65e1 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -576,6 +576,9 @@ class OSTreeReceiver(object):
if self.repopath is None:
self.repo = OSTree.Repo.new_default()
+ self.repopath = self.repo.get_path().get_path()
+ # NOTE: OSTree.Repo.get_path() returns Gio.File
+ # Gio.File.get_path() returns a string of the pathway
else:
self.repo = OSTree.Repo.new(Gio.File.new_for_path(self.repopath))
self.repo.open(None)
[buildstream] 12/15: pushreceive.py: Add comments to code and name
variables more sensibly
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c33fcc2bd0dbb1a6b7a3c3af09aece95b6096943
Author: James Ennis <ja...@codethink.com>
AuthorDate: Fri Apr 20 17:20:35 2018 +0100
pushreceive.py: Add comments to code and name variables more sensibly
---
buildstream/_artifactcache/pushreceive.py | 38 +++++++++++++++++++++++--------
1 file changed, 29 insertions(+), 9 deletions(-)
diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index 57a4b4e..a8ba742 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -114,7 +114,7 @@ class PushMessageWriter(object):
def __init__(self, file, byteorder=sys.byteorder):
self.file = file
self.byteorder = byteorder
- self.msg_byteorder = python_to_msg_byteorder(self.byteorder)
+ self.msg_byteorder = python_to_msg_byteorder(self.byteorder) # 'l' or 'B'
def encode_header(self, cmdtype, size):
header = self.msg_byteorder.encode() + \
@@ -177,6 +177,7 @@ class PushMessageWriter(object):
def send_update(self, refs):
cmdtype = PushCommandType.update
args = {}
+ # Note: Here, revs will be a tuple of (remote_rev, local_rev)
for branch, revs in refs.items():
args[branch] = GLib.Variant('(ss)', revs)
command = PushCommand(cmdtype, args)
@@ -193,14 +194,19 @@ class PushMessageWriter(object):
# Open a TarFile for writing uncompressed tar to a stream
tar = tarfile.TarFile.open(mode='w|', fileobj=self.file)
+ # Note: objects is a set containing: <commit>.GLib.Enum
+ # for each object
for obj in objects:
logging.info('Sending object {}'.format(obj))
+ # obtain the path of the object
+ # Example: path/to/ostree/cache/objects/4a/COMMIT.GlibEnum
objpath = ostree_object_path(repo, obj)
stat = os.stat(objpath)
tar_info = tarfile.TarInfo(obj)
tar_info.mtime = stat.st_mtime
+ # obtain the size of the file in bytes
tar_info.size = stat.st_size
with open(objpath, 'rb') as obj_fp:
tar.addfile(tar_info, obj_fp)
@@ -381,6 +387,7 @@ def foo_run(func, args, stdin_fd, stdout_fd, stderr_fd):
class ProcessWithPipes(object):
def __init__(self, func, args, *, stderr=None):
+ # Create a pipe and return a pair of file descriptors (r, w)
r0, w0 = os.pipe()
r1, w1 = os.pipe()
if stderr is None:
@@ -425,10 +432,12 @@ class OSTreePusher(object):
# Enumerate branches to push
if branches is None:
+ # obtain a dict of 'refs': 'checksums'
_, self.refs = self.repo.list_refs(None, None)
else:
self.refs = {}
for branch in branches:
+ # branch is a ref, now find its checksum (i.e. rev)
_, rev = self.repo.resolve_rev(branch, False)
self.refs[branch] = rev
@@ -447,6 +456,9 @@ class OSTreePusher(object):
logging.info('Executing {}'.format(' '.join(ssh_cmd)))
if self.remote_host:
+ # subprocess.Popen(args, bufsize=-1,...)
+ # Executes a child program in a new process which returns an open file
+ # object connected to the pipe.
self.ssh = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=self.output,
@@ -457,31 +469,33 @@ class OSTreePusher(object):
self.writer = PushMessageWriter(self.ssh.stdin)
self.reader = PushMessageReader(self.ssh.stdout)
- def needed_commits(self, remote, local, needed):
- parent = local
- if remote == '0' * 64:
+ def needed_commits(self, remote_rev, local_rev, needed):
+ parent = local_rev
+ if remote_rev == '0' * 64:
# Nonexistent remote branch, use None for convenience
- remote = None
- while parent != remote:
+ remote_rev = None
+ while parent != remote_rev:
needed.add(parent)
_, commit = self.repo.load_variant_if_exists(OSTree.ObjectType.COMMIT,
parent)
if commit is None:
raise PushException('Shallow history from commit {} does '
- 'not contain remote commit {}'.format(local, remote))
+ 'not contain remote commit {}'.format(local_rev, remote_rev))
parent = OSTree.commit_get_parent(commit)
if parent is None:
break
- if remote is not None and parent != remote:
+ if remote_rev is not None and parent != remote_rev:
self.writer.send_done()
raise PushExistsException('Remote commit {} not descendent of '
- 'commit {}'.format(remote, local))
+ 'commit {}'.format(remote_rev, local_rev))
def needed_objects(self, commits):
objects = set()
for rev in commits:
+ # obtain a set containing all reachable objects from a specific rev
_, reachable = self.repo.traverse_commit(rev, 0, None)
for obj in reachable:
+ # Get the objects name. General form: <commit>.GLib.Enum
objname = OSTree.object_to_string(obj[0], obj[1])
if obj[1] == OSTree.ObjectType.FILE:
# Make this a filez since we're archive-z2
@@ -523,6 +537,8 @@ class OSTreePusher(object):
for branch, rev in self.refs.items():
remote_rev = remote_refs.get(branch, '0' * 64)
if rev != remote_rev:
+ # if the checksums for a branch aren't equal add a tuple of
+ # the remote_rev and local rev to a new dictionary.
update_refs[branch] = remote_rev, rev
if not update_refs:
logging.info('Nothing to update')
@@ -543,9 +559,12 @@ class OSTreePusher(object):
commits = set()
exc_info = None
ref_count = 0
+
+ # update the remote checksum with the local one
for branch, revs in update_refs.items():
logging.info('Updating {} {} to {}'.format(branch, revs[0], revs[1]))
try:
+ # obtain a set of the commits needed to be pushed
self.needed_commits(revs[0], revs[1], commits)
ref_count += 1
except PushExistsException:
@@ -557,6 +576,7 @@ class OSTreePusher(object):
raise exc_info[0].with_traceback(exc_info[1], exc_info[2])
logging.info('Enumerating objects to send')
+ # obtain a set of the objects which need to be pushed to the server
objects = self.needed_objects(commits)
# Send all the objects to receiver, checking status after each
[buildstream] 11/15: push.py: Add tests to test remote cache expiry
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7c194feaf681f3896799f65b99d518318b0378ac
Author: James Ennis <ja...@codethink.com>
AuthorDate: Wed Apr 25 10:47:21 2018 +0100
push.py: Add tests to test remote cache expiry
The tests include a new environment variable: BST_CACHE_QUOTA.
This variable is used in receive_main() in pushreceive.py.
Test names: test_artifact_expires, test_large_artifact_is_not_cached_remotely
test_recently_pulled_artifact_does_not_expire: marked as xfail until
we implement LRU expiry in remote share
---
buildstream/_artifactcache/pushreceive.py | 4 +
tests/frontend/push.py | 164 +++++++++++++++++++++++++++++-
2 files changed, 167 insertions(+), 1 deletion(-)
diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index c60011d..57a4b4e 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -871,5 +871,9 @@ def receive_main(verbose, debug, pull_url, cache_quota, repo):
logging.basicConfig(format='%(module)s: %(levelname)s: %(message)s',
level=loglevel, stream=sys.stderr)
+ # Use environment variable for the tests
+ if not cache_quota and 'BST_TEST_CACHE_QUOTA' in os.environ:
+ cache_quota = os.getenv('BST_TEST_CACHE_QUOTA')
+
receiver = OSTreeReceiver(repo, pull_url, cache_quota)
return receiver.run()
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index e9c2daa..e23d134 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -2,7 +2,7 @@ import os
import shutil
import pytest
from buildstream._exceptions import ErrorDomain
-from tests.testutils import cli, create_artifact_share
+from tests.testutils import cli, create_artifact_share, create_element_size
from tests.testutils.site import IS_LINUX
# Project directory
@@ -195,3 +195,165 @@ def test_push_after_pull(cli, tmpdir, datafiles):
result.assert_success()
assert result.get_pulled_elements() == ['target.bst']
assert result.get_pushed_elements() == ['target.bst']
+
+
+# Ensure that when a pushed artifact's size exceeds the artifact share's
+# quota, the least recently pushed artifact is deleted.
+@pytest.mark.datafiles(DATA_DIR)
+def test_artifact_expires(cli, datafiles, tmpdir):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ element_path = os.path.join(project, 'elements')
+
+ # Create an artifact share (remote artifact cache) in the tmpdir/artifactshare
+ share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'))
+
+ # Set an environment variable of 12 MB for the cache_quota
+ os.environ['BST_TEST_CACHE_QUOTA'] = "12M"
+
+ # Configure bst to push to the cache
+ cli.configure({
+ 'artifacts': {'url': share.repo, 'push': True},
+ })
+
+ # Create and build an element of 5 MB
+ create_element_size('element1.bst', element_path, [], 5000000) # [] => no deps
+ result = cli.run(project=project, args=['build', 'element1.bst'])
+ result.assert_success()
+
+ # Create and build an element of 5 MB
+ create_element_size('element2.bst', element_path, [], 5000000) # [] => no deps
+ result = cli.run(project=project, args=['build', 'element2.bst'])
+ result.assert_success()
+
+ # update the share
+ share.update_summary()
+
+ # check that element's 1 and 2 are cached both locally and remotely
+ assert cli.get_element_state(project, 'element1.bst') == 'cached'
+ assert_shared(cli, share, project, 'element1.bst')
+ assert cli.get_element_state(project, 'element2.bst') == 'cached'
+ assert_shared(cli, share, project, 'element2.bst')
+
+ # Create and build another element of 5 MB (This will exceed cache quota)
+ create_element_size('element3.bst', element_path, [], 5000000)
+ result = cli.run(project=project, args=['build', 'element3.bst'])
+ result.assert_success()
+
+ # update the share
+ share.update_summary()
+
+ # Ensure it is cached both locally and remotely
+ assert cli.get_element_state(project, 'element3.bst') == 'cached'
+ assert_shared(cli, share, project, 'element3.bst')
+
+ # Ensure element1 has been removed from the share
+ assert_not_shared(cli, share, project, 'element1.bst')
+ # Ensure that elemen2 remains
+ assert_shared(cli, share, project, 'element2.bst')
+
+
+# Test that a large artifact, whose size exceeds the quota, is not pushed
+# to the remote share
+@pytest.mark.datafiles(DATA_DIR)
+def test_huge_artifact(cli, datafiles, tmpdir):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ element_path = os.path.join(project, 'elements')
+
+ # Create an artifact share (remote cache) in tmpdir/artifactshare
+ share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'))
+
+ # Set an environment variable of 5 MB for the cache_quota
+ os.environ['BST_TEST_CACHE_QUOTA'] = "5M"
+
+ # Configure bst to push to the cache
+ cli.configure({
+ 'artifacts': {'url': share.repo, 'push': True},
+ })
+
+ create_element_size('small_element.bst', element_path, [], 3000000)
+ result = cli.run(project=project, args=['build', 'small_element.bst'])
+ result.assert_success()
+
+ # Create and build element of 6 MB.
+ create_element_size('large_element.bst', element_path, [], 6000000)
+ result = cli.run(project=project, args=['build', 'large_element.bst'])
+ result.assert_success()
+
+ # update the cache
+ share.update_summary()
+
+ # Ensure that the small artifact is still in the share
+ assert cli.get_element_state(project, 'small_element.bst') == 'cached'
+ assert_shared(cli, share, project, 'small_element.bst')
+
+ # Ensure that the artifact is cached locally but NOT remotely
+ assert cli.get_element_state(project, 'large_element.bst') == 'cached'
+ assert_not_shared(cli, share, project, 'large_element.bst')
+
+
+# Test that when an element is pulled recently, it is not considered the LRU element.
+# NOTE: We expect this test to fail as the current implementation of remote cache
+# expiry only expiries from least recently pushed. NOT least recently used. This will
+# hopefully change when we implement as CAS cache.
+@pytest.mark.xfail
+@pytest.mark.datafiles(DATA_DIR)
+def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ element_path = os.path.join(project, 'elements')
+
+ # Create an artifact share (remote cache) in tmpdir/artifactshare
+ share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'))
+
+ # Set an environment variable of 12 MB for the cache_quota
+ os.environ['BST_TEST_CACHE_QUOTA'] = "12M"
+
+ # Configure bst to push to the cache
+ cli.configure({
+ 'artifacts': {'url': share.repo, 'push': True},
+ })
+
+ # Create and build 2 elements, each of 5 MB.
+ create_element_size('element1.bst', element_path, [], 5000000)
+ result = cli.run(project=project, args=['build', 'element1.bst'])
+ result.assert_success()
+
+ create_element_size('element2.bst', element_path, [], 5000000)
+ result = cli.run(project=project, args=['build', 'element2.bst'])
+ result.assert_success()
+
+ share.update_summary()
+
+ # Ensure they are cached locally
+ assert cli.get_element_state(project, 'element1.bst') == 'cached'
+ assert cli.get_element_state(project, 'element2.bst') == 'cached'
+
+ # Ensure that they have been pushed to the cache
+ assert_shared(cli, share, project, 'element1.bst')
+ assert_shared(cli, share, project, 'element2.bst')
+
+ # Remove element1 from the local cache
+ cli.remove_artifact_from_cache(project, 'element1.bst')
+ assert cli.get_element_state(project, 'element1.bst') != 'cached'
+
+ # Pull the first element from the remote (this should update its mtime)
+ result = cli.run(project=project, args=['pull', 'element1.bst', '--remote',
+ share.repo])
+ result.assert_success()
+
+ # Ensure element1 is cached locally
+ assert cli.get_element_state(project, 'element1.bst') == 'cached'
+
+ # Create and build the element3 (of 5 MB)
+ create_element_size('element3.bst', element_path, [], 5000000)
+ result = cli.run(project=project, args=['build', 'element3.bst'])
+ result.assert_success()
+
+ share.update_summary()
+
+ # Make sure it's cached locally and remotely
+ assert cli.get_element_state(project, 'element3.bst') == 'cached'
+ assert_shared(cli, share, project, 'element3.bst')
+
+ # Ensure that element2 was deleted from the share and element1 remains
+ assert_not_shared(cli, share, project, 'element2.bst')
+ assert_shared(cli, share, project, 'element1.bst')